ARROW-16474: [C++][Packaging] Require Python 3.7 or later

Closes apache#13079 from kou/packaging-linux-drop-python-3.6-support Authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
milvus-io · May 7, 2022 · d8173f7 · d8173f7
1 parent 3a646b3
commit d8173f7
Show file tree

Hide file tree

Showing 12 changed files with 127 additions and 107 deletions.
diff --git a/cpp/cmake_modules/FindBoostAlt.cmake b/cpp/cmake_modules/FindBoostAlt.cmake
@@ -21,15 +21,15 @@ if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT)
   set(Boost_NO_SYSTEM_PATHS ON)
 endif()
 
-set(BoostAlt_FIND_VERSION_OPTIONS)
+set(BoostAlt_FIND_PACKAGE_OPTIONS)
 if(BoostAlt_FIND_VERSION)
-  list(APPEND BoostAlt_FIND_VERSION_OPTIONS ${BoostAlt_FIND_VERSION})
+  list(APPEND BoostAlt_FIND_PACKAGE_OPTIONS ${BoostAlt_FIND_VERSION})
 endif()
 if(BoostAlt_FIND_REQUIRED)
-  list(APPEND BoostAlt_FIND_VERSION_OPTIONS REQUIRED)
+  list(APPEND BoostAlt_FIND_PACKAGE_OPTIONS REQUIRED)
 endif()
 if(BoostAlt_FIND_QUIETLY)
-  list(APPEND BoostAlt_FIND_VERSION_OPTIONS QUIET)
+  list(APPEND BoostAlt_FIND_PACKAGE_OPTIONS QUIET)
 endif()
 
 if(ARROW_BOOST_USE_SHARED)
@@ -38,14 +38,14 @@ if(ARROW_BOOST_USE_SHARED)
   set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS})
   set(BUILD_SHARED_LIBS ON)
 
-  find_package(Boost ${BoostAlt_FIND_VERSION_OPTIONS} COMPONENTS system filesystem)
+  find_package(Boost ${BoostAlt_FIND_PACKAGE_OPTIONS} COMPONENTS system filesystem)
   set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP})
   unset(BUILD_SHARED_LIBS_KEEP)
 else()
   # Find static boost headers and libs
   # TODO Differentiate here between release and debug builds
   set(Boost_USE_STATIC_LIBS ON)
-  find_package(Boost ${BoostAlt_FIND_VERSION_OPTIONS} COMPONENTS system filesystem)
+  find_package(Boost ${BoostAlt_FIND_PACKAGE_OPTIONS} COMPONENTS system filesystem)
 endif()
 
 if(Boost_FOUND)

diff --git a/cpp/cmake_modules/FindPython3Alt.cmake b/cpp/cmake_modules/FindPython3Alt.cmake
@@ -23,16 +23,24 @@
 # - PYTHON_OTHER_LIBS
 # - NUMPY_INCLUDE_DIRS
 
+set(Python3Alt_FIND_PACKAGE_OPTIONS)
+set(Python3Alt_NumPy_FIND_PACKAGE_OPTIONS)
+if(Python3Alt_FIND_VERSION)
+  list(APPEND Python3Alt_FIND_PACKAGE_OPTIONS ${Python3Alt_FIND_VERSION})
+endif()
+if(Python3Alt_FIND_REQUIRED)
+  list(APPEND Python3Alt_FIND_PACKAGE_OPTIONS REQUIRED)
+  list(APPEND Python3Alt_NumPy_FIND_PACKAGE_OPTIONS REQUIRED)
+endif()
+if(Python3Alt_FIND_QUIETLY)
+  list(APPEND Python3Alt_FIND_PACKAGE_OPTIONS QUIET)
+  list(APPEND Python3Alt_NumPy_FIND_PACKAGE_OPTIONS QUIET)
+endif()
+
 # Need CMake 3.15 or later for Python3_FIND_STRATEGY
 if(${CMAKE_VERSION} VERSION_LESS "3.15.0")
-  # Use deprecated Python- and NumPy-finding code
-  if(Python3Alt_FIND_REQUIRED)
-    find_package(PythonLibsNew REQUIRED)
-    find_package(NumPy REQUIRED)
-  else()
-    find_package(PythonLibsNew)
-    find_package(NumPy)
-  endif()
+  find_package(PythonLibsNew ${Python3Alt_FIND_PACKAGE_OPTIONS})
+  find_package(NumPy ${Python3Alt_NumPy_FIND_PACKAGE_OPTIONS})
   find_package_handle_standard_args(
     Python3Alt REQUIRED_VARS PYTHON_EXECUTABLE PYTHON_INCLUDE_DIRS NUMPY_INCLUDE_DIRS)
   return()
@@ -42,21 +50,11 @@ if(${CMAKE_VERSION} VERSION_LESS "3.18.0" OR ARROW_BUILD_TESTS)
   # When building arrow-python-test, we need libpython to be present, so ask for
   # the full "Development" component.  Also ask for it on CMake < 3.18,
   # where "Development.Module" is not available.
-  if(Python3Alt_FIND_REQUIRED)
-    find_package(Python3
-                 COMPONENTS Interpreter Development NumPy
-                 REQUIRED)
-  else()
-    find_package(Python3 COMPONENTS Interpreter Development NumPy)
-  endif()
+  find_package(Python3 ${Python3Alt_FIND_PACKAGE_OPTIONS} COMPONENTS Interpreter
+                                                                     Development NumPy)
 else()
-  if(Python3Alt_FIND_REQUIRED)
-    find_package(Python3
-                 COMPONENTS Interpreter Development.Module NumPy
-                 REQUIRED)
-  else()
-    find_package(Python3 COMPONENTS Interpreter Development.Module NumPy)
-  endif()
+  find_package(Python3 ${Python3Alt_FIND_PACKAGE_OPTIONS}
+               COMPONENTS Interpreter Development.Module NumPy)
 endif()
 
 if(NOT Python3_FOUND)

diff --git a/cpp/src/arrow/python/CMakeLists.txt b/cpp/src/arrow/python/CMakeLists.txt
@@ -19,7 +19,7 @@
 # arrow_python
 #
 
-find_package(Python3Alt REQUIRED)
+find_package(Python3Alt 3.7 REQUIRED)
 
 add_custom_target(arrow_python-all)
 add_custom_target(arrow_python)

diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh
@@ -63,6 +63,7 @@ esac
 
 have_flight=yes
 have_plasma=yes
+have_python=yes
 workaround_missing_packages=()
 case "${distribution}-${code_name}" in
   debian-*)
@@ -71,6 +72,9 @@ case "${distribution}-${code_name}" in
       -e "s/ main$/ main contrib non-free/g" \
       /etc/apt/sources.list
     ;;
+  ubuntu-bionic)
+    have_python=no
+    ;;
 esac
 if [ "$(arch)" = "aarch64" ]; then
   have_plasma=no
@@ -173,9 +177,11 @@ if [ "${have_flight}" = "yes" ]; then
 fi
 
 
-echo "::group::Test libarrow-python"
-${APT_INSTALL} libarrow-python-dev=${package_version}
-echo "::endgroup::"
+if [ "${have_python}" = "yes" ]; then
+  echo "::group::Test libarrow-python"
+  ${APT_INSTALL} libarrow-python-dev=${package_version}
+  echo "::endgroup::"
+fi
 
 
 if [ "${have_plasma}" = "yes" ]; then

diff --git a/dev/release/verify-yum.sh b/dev/release/verify-yum.sh
@@ -73,6 +73,7 @@ case "${distribution}-${distribution_version}" in
     cmake_command=cmake3
     have_flight=no
     have_gandiva=no
+    have_python=no
     install_command="yum install -y"
     ;;
   centos-*)

diff --git a/dev/tasks/linux-packages/apache-arrow/Rakefile b/dev/tasks/linux-packages/apache-arrow/Rakefile
@@ -116,6 +116,16 @@ class ApacheArrowPackageTask < PackageTask
     control.gsub(/@USE_SYSTEM_GRPC@/, use_system_grpc)
   end
 
+  def apt_prepare_debian_control_python(control, target)
+    case target
+    when /\Aubuntu-bionic/
+      enable_python = "#"
+    else
+      enable_python = ""
+    end
+    control.gsub(/@ENABLE_PYTHON@/, enable_python)
+  end
+
   def apt_prepare_debian_control_thrift(control, target)
     case target
     when /\Aubuntu-bionic/
@@ -151,6 +161,7 @@ class ApacheArrowPackageTask < PackageTask
     control = apt_prepare_debian_control_cuda_architecture(control, target)
     control = apt_prepare_debian_control_c_ares(control, target)
     control = apt_prepare_debian_control_grpc(control, target)
+    control = apt_prepare_debian_control_python(control, target)
     control = apt_prepare_debian_control_thrift(control, target)
     control = apt_prepare_debian_control_utf8proc(control, target)
     control = apt_prepare_debian_control_zstd(control, target)

diff --git a/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile b/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile
@@ -58,7 +58,6 @@ RUN \
     nlohmann-json-dev \
     pkg-config \
     python3-dev \
-    python3-numpy \
     python3-pip \
     python3-setuptools \
     python3-wheel \

diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in
@@ -32,8 +32,8 @@ Build-Depends:
   nvidia-cuda-toolkit [!arm64],
   pkg-config,
 @USE_SYSTEM_GRPC@  protobuf-compiler-grpc,
-  python3-dev,
-  python3-numpy,
+@ENABLE_PYTHON@  python3-dev,
+@ENABLE_PYTHON@  python3-numpy,
   tzdata,
   zlib1g-dev
 Build-Depends-Indep: libglib2.0-doc
@@ -92,34 +92,34 @@ Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ library files for Flight RPC system.
 
-Package: libarrow-python900
-Section: libs
-Architecture: any
-Multi-Arch: same
-Pre-Depends: ${misc:Pre-Depends}
-Depends:
-  ${misc:Depends},
-  ${shlibs:Depends},
-  libarrow900 (= ${binary:Version}),
-  python3,
-  python3-numpy
-Description: Apache Arrow is a data processing library for analysis
- .
- This package provides C++ library files for Python support.
-
-Package: libarrow-python-flight900
-Section: libs
-Architecture: any
-Multi-Arch: same
-Pre-Depends: ${misc:Pre-Depends}
-Depends:
-  ${misc:Depends},
-  ${shlibs:Depends},
-  libarrow-flight900 (= ${binary:Version}),
-  libarrow-python900 (= ${binary:Version})
-Description: Apache Arrow is a data processing library for analysis
- .
- This package provides C++ library files for Flight and Python support.
+@ENABLE_PYTHON@Package: libarrow-python900
+@ENABLE_PYTHON@Section: libs
+@ENABLE_PYTHON@Architecture: any
+@ENABLE_PYTHON@Multi-Arch: same
+@ENABLE_PYTHON@Pre-Depends: ${misc:Pre-Depends}
+@ENABLE_PYTHON@Depends:
+@ENABLE_PYTHON@  ${misc:Depends},
+@ENABLE_PYTHON@  ${shlibs:Depends},
+@ENABLE_PYTHON@  libarrow900 (= ${binary:Version}),
+@ENABLE_PYTHON@  python3,
+@ENABLE_PYTHON@  python3-numpy
+@ENABLE_PYTHON@Description: Apache Arrow is a data processing library for analysis
+@ENABLE_PYTHON@ .
+@ENABLE_PYTHON@ This package provides C++ library files for Python support.
+
+@ENABLE_PYTHON@Package: libarrow-python-flight900
+@ENABLE_PYTHON@Section: libs
+@ENABLE_PYTHON@Architecture: any
+@ENABLE_PYTHON@Multi-Arch: same
+@ENABLE_PYTHON@Pre-Depends: ${misc:Pre-Depends}
+@ENABLE_PYTHON@Depends:
+@ENABLE_PYTHON@  ${misc:Depends},
+@ENABLE_PYTHON@  ${shlibs:Depends},
+@ENABLE_PYTHON@  libarrow-flight900 (= ${binary:Version}),
+@ENABLE_PYTHON@  libarrow-python900 (= ${binary:Version})
+@ENABLE_PYTHON@Description: Apache Arrow is a data processing library for analysis
+@ENABLE_PYTHON@ .
+@ENABLE_PYTHON@ This package provides C++ library files for Flight and Python support.
 
 Package: libarrow-dev
 Section: libdevel
@@ -183,30 +183,30 @@ Description: Apache Arrow is a data processing library for analysis
  .
  This package provides C++ header files for Flight RPC system.
 
-Package: libarrow-python-dev
-Section: libdevel
-Architecture: any
-Multi-Arch: same
-Depends:
-  ${misc:Depends},
-  libarrow-dev (= ${binary:Version}),
-  libarrow-python900 (= ${binary:Version})
-Description: Apache Arrow is a data processing library for analysis
- .
- This package provides C++ header files for Python support.
-
-Package: libarrow-python-flight-dev
-Section: libdevel
-Architecture: any
-Multi-Arch: same
-Depends:
-  ${misc:Depends},
-  libarrow-flight-dev (= ${binary:Version}),
-  libarrow-python-dev (= ${binary:Version}),
-  libarrow-python-flight900 (= ${binary:Version})
-Description: Apache Arrow is a data processing library for analysis
- .
- This package provides C++ header files for Flight and Python support.
+@ENABLE_PYTHON@Package: libarrow-python-dev
+@ENABLE_PYTHON@Section: libdevel
+@ENABLE_PYTHON@Architecture: any
+@ENABLE_PYTHON@Multi-Arch: same
+@ENABLE_PYTHON@Depends:
+@ENABLE_PYTHON@  ${misc:Depends},
+@ENABLE_PYTHON@  libarrow-dev (= ${binary:Version}),
+@ENABLE_PYTHON@  libarrow-python900 (= ${binary:Version})
+@ENABLE_PYTHON@Description: Apache Arrow is a data processing library for analysis
+@ENABLE_PYTHON@ .
+@ENABLE_PYTHON@ This package provides C++ header files for Python support.
+
+@ENABLE_PYTHON@Package: libarrow-python-flight-dev
+@ENABLE_PYTHON@Section: libdevel
+@ENABLE_PYTHON@Architecture: any
+@ENABLE_PYTHON@Multi-Arch: same
+@ENABLE_PYTHON@Depends:
+@ENABLE_PYTHON@  ${misc:Depends},
+@ENABLE_PYTHON@  libarrow-flight-dev (= ${binary:Version}),
+@ENABLE_PYTHON@  libarrow-python-dev (= ${binary:Version}),
+@ENABLE_PYTHON@  libarrow-python-flight900 (= ${binary:Version})
+@ENABLE_PYTHON@Description: Apache Arrow is a data processing library for analysis
+@ENABLE_PYTHON@ .
+@ENABLE_PYTHON@ This package provides C++ header files for Flight and Python support.
 
 Package: libgandiva900
 Section: libs

diff --git a/dev/tasks/linux-packages/apache-arrow/debian/rules b/dev/tasks/linux-packages/apache-arrow/debian/rules
@@ -21,22 +21,33 @@ override_dh_auto_configure:
 	  ARROW_CUDA=OFF;					\
 	  ARROW_PLASMA=OFF;					\
 	fi;							\
+	if python3 -c 'import numpy' > /dev/null 2>&1; then	\
+	  ARROW_PYTHON=ON;					\
+	else							\
+	  ARROW_PYTHON=OFF;					\
+	fi;							\
 	dh_auto_configure					\
 	  --sourcedirectory=cpp					\
 	  --builddirectory=cpp_build				\
 	  --buildsystem=cmake+ninja				\
 	  --							\
+	  -DARROW_COMPUTE=ON					\
+	  -DARROW_CSV=ON					\
 	  -DARROW_CUDA=$${ARROW_CUDA}				\
+	  -DARROW_DATASET=ON					\
+	  -DARROW_FILESYSTEM=ON					\
 	  -DARROW_FLIGHT=ON					\
 	  -DARROW_GANDIVA=ON					\
 	  -DARROW_GANDIVA_JAVA=OFF				\
 	  -DARROW_GCS=ON					\
+	  -DARROW_HDFS=ON					\
+	  -DARROW_JSON=ON					\
 	  -DARROW_MIMALLOC=ON					\
 	  -DARROW_ORC=ON					\
 	  -DARROW_PACKAGE_KIND=deb				\
 	  -DARROW_PARQUET=ON					\
 	  -DARROW_PLASMA=$${ARROW_PLASMA}			\
-	  -DARROW_PYTHON=ON					\
+	  -DARROW_PYTHON=$${ARROW_PYTHON}			\
 	  -DARROW_S3=ON						\
 	  -DARROW_USE_CCACHE=OFF				\
 	  -DARROW_WITH_BROTLI=ON				\

diff --git a/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile b/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile
@@ -51,10 +51,10 @@ RUN \
     ninja-build \
     openssl-devel \
     pkg-config \
-    python3 \
-    python3-devel \
-    python3-numpy \
-    python3-pip \
+    python39 \
+    python39-devel \
+    python39-numpy \
+    python39-pip \
     re2-devel \
     # rapidjson-devel \
     rpmdevtools \