TVM instructions: SECP256K1_XONLY_PUBKEY_TWEAK_ADD, SETCONTCTRMANY(X) (…

…ton-blockchain#1404) * TVM instructions: SECP256K1_XONLY_PUBKEY_TWEAK_ADD, SETCONTCTRMANY(X) * Add tests for xonly_pubkey_tweak_add * added secp256k1 as submodule, since we need extrakeys feature of secp256k1 * cleanup * add ton_crypto_core secp256k1 dependency * adjust Dockerfile, android and wasm builds * adjust nix build * test windows build with SECP256K1_ENABLE_MODULE_EXTRAKEYS * test windows build with SECP256K1_ENABLE_MODULE_EXTRAKEYS * adjust android build * adjust emscripten build * adjust emscripten build * try macos-13 * emscripten build adjustments * windows build adjustments * final corrections --------- Co-authored-by: neodix <[email protected]>
ChorusOne · Nov 26, 2024 · 25b4c67 · 25b4c67
1 parent 954a96a
commit 25b4c67
Show file tree

Hide file tree

Showing 56 changed files with 2,115 additions and 505 deletions.
diff --git a/.github/workflows/build-ton-linux-android-tonlib.yml b/.github/workflows/build-ton-linux-android-tonlib.yml
@@ -21,6 +21,8 @@ jobs:
 
     - name: Build TON
       run: |
+        git submodule sync --recursive
+        git submodule update
         cp assembly/android/build-android-tonlib.sh .
         chmod +x build-android-tonlib.sh
         ./build-android-tonlib.sh -a

diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml
@@ -35,6 +35,8 @@ jobs:
 
     - name: Build TON
       run: |
+        git submodule sync --recursive
+        git submodule update
         cp assembly/native/build-ubuntu-shared.sh .
         chmod +x build-ubuntu-shared.sh
         ./build-ubuntu-shared.sh -t -a

diff --git a/.github/workflows/build-ton-macos-15-arm64-shared.yml b/.github/workflows/build-ton-macos-15-arm64-shared.yml
@@ -14,6 +14,8 @@ jobs:
 
       - name: Build TON
         run: |
+          git submodule sync --recursive
+          git submodule update
           cp assembly/native/build-macos-shared.sh .
           chmod +x build-macos-shared.sh
           ./build-macos-shared.sh -t -a

diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml
@@ -14,6 +14,8 @@ jobs:
 
       - name: Build TON
         run: |
+          git submodule sync --recursive
+          git submodule update
           cp assembly/native/build-macos-shared.sh .
           chmod +x build-macos-shared.sh
           ./build-macos-shared.sh -t -a

diff --git a/.github/workflows/build-ton-macos-x86-64-shared.yml b/.github/workflows/build-ton-macos-x86-64-shared.yml
@@ -4,7 +4,7 @@ on: [push,workflow_dispatch,workflow_call]
 
 jobs:
   build:
-    runs-on: macos-12
+    runs-on: macos-13
 
     steps:
       - name: Check out repository
@@ -14,12 +14,14 @@ jobs:
 
       - name: Build TON
         run: |
+          git submodule sync --recursive
+          git submodule update
           cp assembly/native/build-macos-shared.sh .
           chmod +x build-macos-shared.sh
           ./build-macos-shared.sh -t -a
 
       - name: Upload artifacts
         uses: actions/upload-artifact@master
         with:
-          name: ton-binaries-macos-12
+          name: ton-binaries-macos-13
           path: artifacts
diff --git a/.github/workflows/build-ton-wasm-emscripten.yml b/.github/workflows/build-ton-wasm-emscripten.yml
@@ -15,10 +15,12 @@ jobs:
     - name: Install system libraries
       run: |
         sudo apt-get update
-        sudo apt-get install -y build-essential git openssl cmake ninja-build zlib1g-dev libssl-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev
+        sudo apt-get install -y build-essential git openssl cmake ninja-build zlib1g-dev libssl-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev
 
     - name: Build TON WASM artifacts
       run: |
+        git submodule sync --recursive
+        git submodule update
         cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
         chmod +x fift-func-wasm-build-ubuntu.sh
         ./fift-func-wasm-build-ubuntu.sh -a

diff --git a/.github/workflows/ton-arm64-macos.yml b/.github/workflows/ton-arm64-macos.yml
@@ -18,6 +18,8 @@ jobs:
 
     - name: Build TON
       run: |
+        git submodule sync --recursive
+        git submodule update
         cp assembly/nix/build-macos-nix.sh .
         chmod +x build-macos-nix.sh
         ./build-macos-nix.sh -t

diff --git a/.github/workflows/ton-x86-64-linux.yml b/.github/workflows/ton-x86-64-linux.yml
@@ -22,6 +22,8 @@ jobs:
 
     - name: Build TON
       run: |
+        git submodule sync --recursive
+        git submodule update
         cp assembly/nix/build-linux-x86-64-nix.sh .
         chmod +x build-linux-x86-64-nix.sh
         ./build-linux-x86-64-nix.sh -t

diff --git a/.github/workflows/ton-x86-64-macos.yml b/.github/workflows/ton-x86-64-macos.yml
@@ -4,7 +4,7 @@ on: [push,workflow_dispatch,workflow_call]
 
 jobs:
   build:
-    runs-on: macos-12
+    runs-on: macos-13
 
     steps:
     - uses: actions/checkout@v3
@@ -18,6 +18,8 @@ jobs:
 
     - name: Build TON
       run: |
+        git submodule sync --recursive
+        git submodule update
         cp assembly/nix/build-macos-nix.sh .
         chmod +x build-macos-nix.sh
         ./build-macos-nix.sh -t

diff --git a/.github/workflows/ton-x86-64-windows.yml b/.github/workflows/ton-x86-64-windows.yml
@@ -23,6 +23,8 @@ jobs:
 
       - name: Build TON
         run: |
+          git submodule sync --recursive
+          git submodule update
           copy assembly\native\build-windows-github-2019.bat .
           copy assembly\native\build-windows-2019.bat .
           build-windows-github-2019.bat Enterprise

diff --git a/.gitignore b/.gitignore
@@ -19,7 +19,6 @@ zlib/
 libsodium/
 libmicrohttpd-0.9.77-w32-bin/
 readline-5.0-1-lib/
-secp256k1/
 openssl-3.1.4/
 libsodium-1.0.18-stable-msvc.zip
 libmicrohttpd-0.9.77-w32-bin.zip

diff --git a/.gitmodules b/.gitmodules
@@ -13,3 +13,7 @@
 [submodule "third-party/blst"]
 	path = third-party/blst
 	url = https://github.com/supranational/blst.git
+[submodule "third-party/secp256k1"]
+	path = third-party/secp256k1
+	url = https://github.com/bitcoin-core/secp256k1
+	branch = v0.3.2
diff --git a/CMake/BuildSECP256K1.cmake b/CMake/BuildSECP256K1.cmake
@@ -0,0 +1,54 @@
+if (NOT SECP256K1_LIBRARY)
+
+    set(SECP256K1_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third-party/secp256k1)
+    set(SECP256K1_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/third-party/secp256k1)
+    set(SECP256K1_INCLUDE_DIR ${SECP256K1_BINARY_DIR}/include)
+
+    file(MAKE_DIRECTORY ${SECP256K1_BINARY_DIR})
+
+    if (MSVC)
+      set(SECP256K1_BINARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third-party/secp256k1)
+      set(SECP256K1_LIBRARY ${SECP256K1_SOURCE_DIR}/build/src/Release/libsecp256k1.lib)
+      set(SECP256K1_INCLUDE_DIR ${SECP256K1_BINARY_DIR}/include)
+      add_custom_command(
+        WORKING_DIRECTORY ${SECP256K1_SOURCE_DIR}
+        COMMAND cmake -E env CFLAGS="/WX" cmake -A x64 -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DSECP256K1_ENABLE_MODULE_EXTRAKEYS=ON -DSECP256K1_BUILD_EXAMPLES=OFF -DBUILD_SHARED_LIBS=OFF
+        COMMAND cmake --build build --config Release
+        COMMENT "Build Secp256k1"
+        DEPENDS ${SECP256K1_SOURCE_DIR}
+        OUTPUT ${SECP256K1_LIBRARY}
+      )
+    elseif (EMSCRIPTEN)
+      set(SECP256K1_BINARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third-party/secp256k1)
+      set(SECP256K1_LIBRARY ${SECP256K1_BINARY_DIR}/.libs/libsecp256k1.a)
+      set(SECP256K1_INCLUDE_DIR ${SECP256K1_SOURCE_DIR}/include)
+      add_custom_command(
+          WORKING_DIRECTORY ${SECP256K1_SOURCE_DIR}
+          COMMAND ./autogen.sh
+          COMMAND emconfigure ./configure --enable-module-recovery --enable-module-extrakeys --disable-tests --disable-benchmark
+          COMMAND emmake make clean
+          COMMAND emmake make
+          COMMENT "Build Secp256k1 with emscripten"
+          DEPENDS ${SECP256K1_SOURCE_DIR}
+          OUTPUT ${SECP256K1_LIBRARY}
+      )
+    else()
+      if (NOT NIX)
+        set(SECP256K1_LIBRARY ${SECP256K1_BINARY_DIR}/lib/libsecp256k1.a)
+        add_custom_command(
+            WORKING_DIRECTORY ${SECP256K1_SOURCE_DIR}
+            COMMAND ./autogen.sh
+            COMMAND ./configure -q --disable-option-checking --enable-module-recovery --enable-module-extrakeys --prefix ${SECP256K1_BINARY_DIR} --with-pic --disable-shared --enable-static --disable-tests --disable-benchmark
+            COMMAND make -j16
+            COMMAND make install
+            COMMENT "Build secp256k1"
+            DEPENDS ${SECP256K1_SOURCE_DIR}
+            OUTPUT ${SECP256K1_LIBRARY}
+        )
+      endif()
+    endif()
+else()
+   message(STATUS "Use Secp256k1: ${SECP256K1_LIBRARY}")
+endif()
+
+add_custom_target(secp256k1 DEPENDS ${SECP256K1_LIBRARY})
diff --git a/CMake/FindSecp256k1.cmake b/CMake/FindSecp256k1.cmake
@@ -1,9 +1,8 @@
-# - Try to find SECP256K1
+# - Try to find Secp256k1
 # Once done this will define
 #
-#  SECP256K1_FOUND - system has SECP256K1
-#  SECP256K1_INCLUDE_DIR - the SECP256K1 include directory
-#  SECP256K1_LIBRARY - Link these to use SECP256K1
+#  SECP256K1_INCLUDE_DIR - the Secp256k1 include directory
+#  SECP256K1_LIBRARY - Link these to use Secp256k1
 
 if (NOT SECP256K1_LIBRARY)
     find_path(

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -183,6 +183,7 @@ message("Add ton")
 set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
 
 include(BuildBLST)
+include(BuildSECP256K1)
 
 # Configure CCache if available
 find_program(CCACHE_FOUND ccache)
@@ -237,6 +238,11 @@ if (TON_USE_JEMALLOC)
   find_package(jemalloc REQUIRED)
 endif()
 
+if (NIX)
+  find_package(Secp256k1 REQUIRED)
+endif()
+
+
 set(MEMPROF "" CACHE STRING "Use one of \"ON\", \"FAST\" or \"SAFE\" to enable memory profiling. \
 Works under macOS and Linux when compiled using glibc. \
 In FAST mode stack is unwinded only using frame pointers, which may fail. \

diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 FROM ubuntu:22.04 AS builder
 RUN apt-get update && \
-        DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev lsb-release software-properties-common gnupg
+        DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev lsb-release software-properties-common gnupg
 
 RUN wget https://apt.llvm.org/llvm.sh && \
     chmod +x llvm.sh && \
@@ -24,7 +24,7 @@ RUN mkdir build && \
 
 FROM ubuntu:22.04
 RUN apt-get update && \
-    apt-get install -y wget curl libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev htop net-tools netcat iptraf-ng jq tcpdump pv plzip && \
+    apt-get install -y wget curl libatomic1 openssl libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev htop net-tools netcat iptraf-ng jq tcpdump pv plzip && \
     rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /var/ton-work/db /var/ton-work/scripts /usr/share/ton/smartcont/ /usr/lib/fift/

diff --git a/assembly/android/build-android-tonlib.sh b/assembly/android/build-android-tonlib.sh
@@ -8,14 +8,19 @@ while getopts 'a' flag; do
   esac
 done
 
+export CC=$(which clang-16)
+export CXX=$(which clang++-16)
+export CCACHE_DISABLE=1
+
 if [ ! -d android-ndk-r25b ]; then
   rm android-ndk-r25b-linux.zip
+  echo "Downloading https://dl.google.com/android/repository/android-ndk-r25b-linux.zip"
   wget -q https://dl.google.com/android/repository/android-ndk-r25b-linux.zip
   unzip -q android-ndk-r25b-linux.zip
   test $? -eq 0 || { echo "Can't unzip android-ndk-r25b-linux.zip"; exit 1; }
-  echo Android NDK extracted
+  echo "Android NDK extracted"
 else
-  echo Using extracted Android NDK
+  echo "Using extracted Android NDK"
 fi
 
 export JAVA_AWT_LIBRARY=NotNeeded

diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh
@@ -52,21 +52,6 @@ else
   echo "Using compiled lz4"
 fi
 
-if [ ! -d "secp256k1" ]; then
-git clone https://github.com/bitcoin-core/secp256k1.git
-cd secp256k1
-secp256k1Path=`pwd`
-git checkout v0.3.2
-./autogen.sh
-./configure --enable-module-recovery --enable-static --disable-tests --disable-benchmark --with-pic
-make -j12
-test $? -eq 0 || { echo "Can't compile secp256k1"; exit 1; }
-cd ..
-else
-  secp256k1Path=$(pwd)/secp256k1
-  echo "Using compiled secp256k1"
-fi
-
 if [ ! -d "libsodium" ]; then
   export LIBSODIUM_FULL_BUILD=1
   git clone https://github.com/jedisct1/libsodium.git
@@ -135,9 +120,6 @@ cmake -GNinja .. \
 -DZLIB_FOUND=1 \
 -DZLIB_INCLUDE_DIR=$zlibPath \
 -DZLIB_LIBRARIES=$zlibPath/libz.a \
--DSECP256K1_FOUND=1 \
--DSECP256K1_INCLUDE_DIR=$secp256k1Path/include \
--DSECP256K1_LIBRARY=$secp256k1Path/.libs/libsecp256k1.a \
 -DSODIUM_FOUND=1 \
 -DSODIUM_INCLUDE_DIR=$sodiumPath/src/libsodium/include \
 -DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \

diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh
@@ -36,21 +36,6 @@ else
 fi
 export CCACHE_DISABLE=1
 
-if [ ! -d "secp256k1" ]; then
-  git clone https://github.com/bitcoin-core/secp256k1.git
-  cd secp256k1
-  secp256k1Path=`pwd`
-  git checkout v0.3.2
-  ./autogen.sh
-  ./configure --enable-module-recovery --enable-static --disable-tests --disable-benchmark
-  make -j12
-  test $? -eq 0 || { echo "Can't compile secp256k1"; exit 1; }
-  cd ..
-else
-  secp256k1Path=$(pwd)/secp256k1
-  echo "Using compiled secp256k1"
-fi
-
 if [ ! -d "lz4" ]; then
   git clone https://github.com/lz4/lz4
   cd lz4
@@ -70,9 +55,6 @@ brew unlink openssl@3 &&  brew link --overwrite openssl@3
 
 cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. \
 -DCMAKE_CXX_FLAGS="-stdlib=libc++" \
--DSECP256K1_FOUND=1 \
--DSECP256K1_INCLUDE_DIR=$secp256k1Path/include \
--DSECP256K1_LIBRARY=$secp256k1Path/.libs/libsecp256k1.a \
 -DLZ4_FOUND=1 \
 -DLZ4_LIBRARIES=$lz4Path/lib/liblz4.a \
 -DLZ4_INCLUDE_DIRS=$lz4Path/lib

diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh
@@ -43,23 +43,6 @@ else
   echo "Using compiled lz4"
 fi
 
-if [ ! -d "secp256k1" ]; then
-git clone https://github.com/bitcoin-core/secp256k1.git
-cd secp256k1
-secp256k1Path=`pwd`
-git checkout v0.3.2
-./autogen.sh
-./configure --enable-module-recovery --enable-static --disable-tests --disable-benchmark --with-pic
-make -j12
-test $? -eq 0 || { echo "Can't compile secp256k1"; exit 1; }
-cd ..
-# ./.libs/libsecp256k1.a
-# ./include
-else
-  secp256k1Path=$(pwd)/secp256k1
-  echo "Using compiled secp256k1"
-fi
-
 if [ ! -d "libsodium" ]; then
   export LIBSODIUM_FULL_BUILD=1
   git clone https://github.com/jedisct1/libsodium.git
@@ -126,9 +109,6 @@ cmake -GNinja .. \
 -DZLIB_FOUND=1 \
 -DZLIB_INCLUDE_DIR=$zlibPath \
 -DZLIB_LIBRARIES=$zlibPath/libz.a \
--DSECP256K1_FOUND=1 \
--DSECP256K1_INCLUDE_DIR=$secp256k1Path/include \
--DSECP256K1_LIBRARY=$secp256k1Path/.libs/libsecp256k1.a \
 -DSODIUM_FOUND=1 \
 -DSODIUM_INCLUDE_DIR=$sodiumPath/src/libsodium/include \
 -DSODIUM_LIBRARY_RELEASE=$sodiumPath/src/libsodium/.libs/libsodium.a \

diff --git a/assembly/native/build-windows-2019.bat b/assembly/native/build-windows-2019.bat
@@ -58,26 +58,6 @@ cd ..\..\..\..
 echo Using lz4...
 )
 
-if not exist "secp256k1" (
-git clone https://github.com/bitcoin-core/secp256k1.git
-cd secp256k1
-git checkout v0.3.2
-cmake -G "Visual Studio 16 2019" -A x64 -S . -B build -DSECP256K1_ENABLE_MODULE_RECOVERY=ON -DBUILD_SHARED_LIBS=OFF
-IF %errorlevel% NEQ 0 (
-  echo Can't configure secp256k1
-  exit /b %errorlevel%
-)
-cmake --build build --config Release
-IF %errorlevel% NEQ 0 (
-  echo Can't install secp256k1
-  exit /b %errorlevel%
-)
-cd ..
-) else (
-echo Using secp256k1...
-)
-
-
 curl --retry 5 --retry-delay 10 -Lo libsodium-1.0.18-stable-msvc.zip https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable-msvc.zip
 IF %errorlevel% NEQ 0 (
   echo Can't download libsodium
@@ -131,9 +111,6 @@ cd build
 cmake -GNinja  -DCMAKE_BUILD_TYPE=Release ^
 -DPORTABLE=1 ^
 -DSODIUM_USE_STATIC_LIBS=1 ^
--DSECP256K1_FOUND=1 ^
--DSECP256K1_INCLUDE_DIR=%root%\secp256k1\include ^
--DSECP256K1_LIBRARY=%root%\secp256k1\build\src\Release\libsecp256k1.lib ^
 -DLZ4_FOUND=1 ^
 -DLZ4_INCLUDE_DIRS=%root%\lz4\lib ^
 -DLZ4_LIBRARIES=%root%\lz4\build\VS2017\liblz4\bin\x64_Release\liblz4_static.lib ^