From 7e12e508ff72e62bbd9be8fc613b4b8afdadf163 Mon Sep 17 00:00:00 2001 From: Petr Shumilov Date: Mon, 27 Jan 2025 21:00:31 +0300 Subject: [PATCH] Fix github CI Signed-off-by: Petr Shumilov --- .github/workflows/Build.yml | 4 +-- .github/workflows/Dockerfile.jammy | 13 +++++--- .github/workflows/macos.yml | 2 +- cmake/utils.cmake | 11 ++++--- compiler/compiler-settings.cpp | 43 +++++++++++++------------ runtime/runtime.cmake | 2 +- third-party/curl-cmake/curl.cmake | 13 ++++++-- third-party/openssl-cmake/openssl.cmake | 5 +++ 8 files changed, 57 insertions(+), 36 deletions(-) diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml index 7ce1e24cf1..2c2648aeab 100644 --- a/.github/workflows/Build.yml +++ b/.github/workflows/Build.yml @@ -77,11 +77,11 @@ jobs: - name: Add git safe directory run: docker exec kphp-build-container-${{matrix.os}} bash -c - "git config --global --add safe.directory ${{env.kphp_root_dir}}" + "git config --global --add safe.directory '*'" - name: Build all run: docker exec kphp-build-container-${{matrix.os}} bash -c - "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" + "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=OFF -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all" - name: Run unit tests run: docker exec kphp-build-container-${{matrix.os}} bash -c diff --git a/.github/workflows/Dockerfile.jammy b/.github/workflows/Dockerfile.jammy index 8bbb2ec628..00d320b364 100644 --- a/.github/workflows/Dockerfile.jammy +++ b/.github/workflows/Dockerfile.jammy @@ -16,12 +16,15 @@ RUN apt update && \ add-apt-repository ppa:deadsnakes/ppa && \ apt update && \ apt install -y --no-install-recommends \ - git cmake make g++ lld gperf netcat \ - python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \ - python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \ + build-essential devscripts fakeroot git cmake make g++ lld gperf netcat \ + python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools && \ apt install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \ - libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libnuma-dev unzip \ - libldap-dev libkrb5-dev libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* && \ + libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libnuma-dev unzip && \ + echo "deb https://archive.ubuntu.com/ubuntu focal main universe" >> /etc/apt/sources.list && \ + apt update && \ + apt install -t focal -y --no-install-recommends --allow-downgrades mysql-server libmysqlclient-dev \ + libldap-dev libtinfo6=6.2* ncurses-bin=6.2* libncurses6=6.2* libncursesw6=6.2* libncurses-dev=6.2* libtinfo-dev=6.2* libcom-err2=1.45* comerr-dev=2.1-1.45* libkrb5support0=1.17* libkrb5-3=1.17* libk5crypto3=1.17* libgssapi-krb5-2=1.17* libkrb5-dev=1.17* libpq5=12.* ssl-cert=1.1.* postgresql-common locales=2.35* postgresql-12 libpq-dev=12.* && \ + python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \ rm -rf /var/lib/apt/lists/* # set php7.4 as default diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 1d3f541bf8..06814cf78a 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -36,7 +36,7 @@ jobs: brew install python@3.13 re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4 brew link --overwrite --force shivammathur/php/php@7.4 /opt/homebrew/opt/python@3.13/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/python@3.13/libexec/bin/pip install --break-system-packages jsonschema - + - name: Run cmake run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DDOWNLOAD_MISSING_LIBRARIES=On -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build diff --git a/cmake/utils.cmake b/cmake/utils.cmake index da5d43987a..e5ed59b73d 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -62,12 +62,15 @@ function(update_git_submodules) # Update submodules execute_process( - COMMAND git submodule update --init --recursive - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive + WORKING_DIRECTORY ${BASE_DIR} RESULT_VARIABLE update_result - ERROR_QUIET + OUTPUT_VARIABLE out1 + ERROR_VARIABLE err1 + #ERROR_QUIET ) - + message("out='${out1}'") + message("err='${err1}'") if(NOT update_result EQUAL 0) message(FATAL_ERROR "Failed to update Git submodules.") endif() diff --git a/compiler/compiler-settings.cpp b/compiler/compiler-settings.cpp index 850e201a1e..b859635a77 100644 --- a/compiler/compiler-settings.cpp +++ b/compiler/compiler-settings.cpp @@ -153,7 +153,7 @@ void append_3dparty_lib(std::string &ld_flags, const std::string &path_to_3dpart ld_flags += " " + path_to_3dparty + "lib/lib" + libname + ".a"; } -void append_curl([[maybe_unused]] std::string &cxx_flags, std::string &ld_flags, const std::string &path_to_3dparty) noexcept { +void append_curl([[maybe_unused]] std::string &cxx_flags, std::string &ld_flags, [[maybe_unused]] const std::string &path_to_3dparty) noexcept { if (!contains_lib(ld_flags, "curl")) { #if defined(__APPLE__) ld_flags += " -lcurl"; @@ -367,7 +367,7 @@ void CompilerSettings::init() { ld_flags.value_ = extra_ld_flags.get(); append_curl(cxx_default_flags, ld_flags.value_, third_party_path); append_apple_options(cxx_default_flags, ld_flags.value_); - std::vector os_installed_libs{"pcre", "re2", "yaml-cpp", "h3", "z", "zstd", "nghttp2", "kphp-timelib"}; + std::vector system_installed_static_libs{"pcre", "re2", "yaml-cpp", "h3", "z", "zstd", "nghttp2", "kphp-timelib"}; #ifdef KPHP_TIMELIB_LIB_DIR ld_flags.value_ += " -L" KPHP_TIMELIB_LIB_DIR; @@ -387,46 +387,47 @@ void CompilerSettings::init() { ld_flags.value_ += " -L /usr/local/lib"; #endif - std::vector external_libs{"pthread", "m", "dl"}; + std::vector system_installed_dynamic_libs{"pthread", "m", "dl"}; #ifdef PDO_DRIVER_MYSQL #ifdef PDO_LIBS_STATIC_LINKING - os_installed_libs.emplace_back("mysqlclient"); + system_installed_static_libs.emplace_back("mysqlclient"); #else - external_libs.emplace_back("mysqlclient"); + system_installed_dynamic_libs.emplace_back("mysqlclient"); #endif #endif #ifdef PDO_DRIVER_PGSQL #ifdef PDO_LIBS_STATIC_LINKING ld_flags.value_ += fmt_format(" -L /usr/lib/postgresql/{}/lib/ ", PDO_DRIVER_PGSQL_VERSION); - os_installed_libs.emplace_back("pq"); - os_installed_libs.emplace_back("pgcommon"); - os_installed_libs.emplace_back("pgport"); + system_installed_static_libs.emplace_back("pq"); + system_installed_static_libs.emplace_back("pgcommon"); + system_installed_static_libs.emplace_back("pgport"); // following common libraries are required for libpq.a - external_libs.emplace_back("ldap"); - external_libs.emplace_back("gssapi_krb5"); + system_installed_dynamic_libs.emplace_back("ldap"); + system_installed_dynamic_libs.emplace_back("gssapi_krb5"); #else - external_libs.emplace_back("pq"); + system_installed_dynamic_libs.emplace_back("pq"); #endif #endif - append_3dparty_headers(cxx_default_flags, third_party_path, "openssl"); - append_3dparty_lib(ld_flags.value_, third_party_path, "ssl"); - append_3dparty_lib(ld_flags.value_, third_party_path, "crypto"); - #if defined(__APPLE__) - append_if_doesnt_contain(ld_flags.value_, os_installed_libs, "-l"); + append_if_doesnt_contain(ld_flags.value_, system_installed_static_libs, "-l"); auto flex_prefix = kphp_src_path.value_ + "objs/flex/lib"; append_if_doesnt_contain(ld_flags.value_, vk::to_array({"vk-flex-data"}), flex_prefix, ".a"); - external_libs.emplace_back("iconv"); + system_installed_dynamic_libs.emplace_back("iconv"); #else append_3dparty_lib(ld_flags.value_, third_party_path, "vk-flex-data"); - os_installed_libs.emplace_back("numa"); - append_if_doesnt_contain(ld_flags.value_, os_installed_libs, "-l:lib", ".a"); - external_libs.emplace_back("rt"); + system_installed_static_libs.emplace_back("numa"); + append_if_doesnt_contain(ld_flags.value_, system_installed_static_libs, "-l:lib", ".a"); + system_installed_dynamic_libs.emplace_back("rt"); #endif - append_if_doesnt_contain(ld_flags.value_, external_libs, "-l"); + + append_3dparty_headers(cxx_default_flags, third_party_path, "openssl"); + append_3dparty_lib(ld_flags.value_, third_party_path, "ssl"); + append_3dparty_lib(ld_flags.value_, third_party_path, "crypto"); + + append_if_doesnt_contain(ld_flags.value_, system_installed_dynamic_libs, "-l"); ld_flags.value_ += " -rdynamic"; runtime_headers.value_ = "runtime-headers.h"; diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake index 12b111e055..413624daa6 100644 --- a/runtime/runtime.cmake +++ b/runtime/runtime.cmake @@ -158,7 +158,7 @@ target_link_libraries(kphp-full-runtime PUBLIC ${RUNTIME_LIBS}) set_target_properties(kphp-full-runtime PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${OBJS_DIR}) prepare_cross_platform_libs(RUNTIME_LINK_TEST_LIBS pcre nghttp2 kphp-timelib) -set(RUNTIME_LINK_TEST_LIBS vk::flex_data_static CURL::curl OpenSSL::SSL ${NUMA_LIB} ${RUNTIME_LINK_TEST_LIBS} ${EPOLL_SHIM_LIB} ${ICONV_LIB} ${RT_LIB}) +set(RUNTIME_LINK_TEST_LIBS vk::flex_data_static CURL::curl OpenSSL::SSL ${NUMA_LIB} ${RUNTIME_LINK_TEST_LIBS} ${EPOLL_SHIM_LIB} ${ICONV_LIB} ${RT_LIB} dl) if (PDO_DRIVER_MYSQL) list(APPEND RUNTIME_LINK_TEST_LIBS mysqlclient) diff --git a/third-party/curl-cmake/curl.cmake b/third-party/curl-cmake/curl.cmake index 7d1b6e3683..91d58584a3 100644 --- a/third-party/curl-cmake/curl.cmake +++ b/third-party/curl-cmake/curl.cmake @@ -1,6 +1,16 @@ +execute_process( + COMMAND ${GIT_EXECUTABLE} submodule update --remote third-party/curl + WORKING_DIRECTORY ${BASE_DIR} +) + set(CURL_INSTALL_DIR ${CMAKE_BINARY_DIR}/third-party/curl) file(MAKE_DIRECTORY ${CURL_INSTALL_DIR}) +set(CURL_COMPILE_FLAGS "-Wno-deprecated-declarations") +if(COMPILER_CLANG) + set(CURL_COMPILE_FLAGS "${CURL_COMPILE_FLAGS} -Wno-string-plus-int") +endif() + ExternalProject_Add( curl SOURCE_DIR ${THIRD_PARTY_DIR}/curl @@ -21,8 +31,7 @@ ExternalProject_Add( -DCMAKE_INSTALL_PREFIX=${CURL_INSTALL_DIR} -DCMAKE_INSTALL_LIBDIR=${CURL_INSTALL_DIR}/lib/ -DCMAKE_INSTALL_INCLUDEDIR=${CURL_INSTALL_DIR}/include - -DCMAKE_CXX_FLAGS="-Wno-deprecated-declarations" - -DCMAKE_C_FLAGS="-Wno-deprecated-declarations" + -DCMAKE_C_FLAGS=${CURL_COMPILE_FLAGS} BUILD_COMMAND ${CMAKE_COMMAND} --build . --config $ INSTALL_COMMAND ${CMAKE_COMMAND} --install . --config $ && diff --git a/third-party/openssl-cmake/openssl.cmake b/third-party/openssl-cmake/openssl.cmake index f92170a5e1..fcd2a3d1e5 100644 --- a/third-party/openssl-cmake/openssl.cmake +++ b/third-party/openssl-cmake/openssl.cmake @@ -1,3 +1,8 @@ +execute_process( + COMMAND ${GIT_EXECUTABLE} submodule update --remote third-party/openssl + WORKING_DIRECTORY ${BASE_DIR} +) + set(OPENSSL_INSTALL_DIR ${CMAKE_BINARY_DIR}/third-party/openssl) file(MAKE_DIRECTORY ${OPENSSL_INSTALL_DIR})