From 9f4b43b86d76e68c3cba608ef005e7fb10fa227a Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Mon, 13 Nov 2023 16:07:49 -0800 Subject: [PATCH 1/2] Use f2py directly through cmake (no distutils/setuptools/etc.) --- .github/workflows/MacOS.yml | 4 +- python/CMakeLists.txt | 98 ++++++++++----------------- python/setup.py.in | 29 -------- python/test/{test.py => test_misc.py} | 0 4 files changed, 39 insertions(+), 92 deletions(-) delete mode 100644 python/setup.py.in rename python/test/{test.py => test_misc.py} (100%) diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml index dd3b0afa..f23044ac 100644 --- a/.github/workflows/MacOS.yml +++ b/.github/workflows/MacOS.yml @@ -22,9 +22,11 @@ jobs: - name: install-deps run: | - pip3 install setuptools pip3 install numpy + pip3 install meson + pip3 install ninja pip3 install netCDF4 + pip3 install protobuf sudo ln -sf /usr/local/bin/gfortran-11 /usr/local/bin/gfortran - name: checkout-bufr diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 7fdbe840..d937d4f5 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -1,85 +1,59 @@ # This is the cmake build file for the python directory of the # NCEPLIBS-bufr project. # -# Rahul Mahajan +# Alex Richert, Rahul Mahajan -configure_file(setup.py.in setup.py @ONLY) - -file( COPY ncepbufr utils DESTINATION . ) - -# Library installation directory set(_PYVER "${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}") -set(_lib_dir "${CMAKE_CURRENT_BINARY_DIR}/site-packages") -set(_bin_root "${CMAKE_CURRENT_BINARY_DIR}/bin") -# Build the extension module for use in install tree +file( COPY ncepbufr DESTINATION . ) add_custom_target(python_mod ALL) add_dependencies(python_mod bufr_4) -if( NOT \$ENV{DESTDIR} STREQUAL \"\" ) - set( __root \"--root=\$ENV{DESTDIR}\" ) -endif() add_custom_command( - TARGET python_mod - COMMAND ${Python3_EXECUTABLE} setup.py install - \${__root} - --install-scripts=${_bin_root} - --install-lib=${_lib_dir} - --record=${CMAKE_BINARY_DIR}/extra_install.txt - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS setup.py.in + TARGET python_mod + POST_BUILD + COMMAND ${Python3_EXECUTABLE} -m "numpy.f2py" + -c "${CMAKE_CURRENT_SOURCE_DIR}/_bufrlib.pyf" + -m _bufrlib + -L${CMAKE_BINARY_DIR}/src + -lbufr_4 + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/_bufrlib.pyf" +) + +install( + DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ncepbufr + DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/python${_PYVER}/site-packages +) +install( + DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ + DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/python${_PYVER}/site-packages + FILES_MATCHING + PATTERN "_bufrlib*.so" + PATTERN "CMakeFiles" EXCLUDE +) +install( + PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/utils/prepbufr2nc + DESTINATION ${CMAKE_INSTALL_FULL_BINDIR} ) if(ENABLE_DOCS) # Uses pdoc (https://github.com/BurntSushi/pdoc) find_program(PDOC_EXECUTABLE pdoc REQUIRED) add_custom_target(python_docs ALL - ${CMAKE_COMMAND} -E env PYTHONPATH=${_lib_dir}:$ENV{PYTHONPATH} + ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}:$ENV{PYTHONPATH} ${PDOC_EXECUTABLE} -o ../docs/html/python ./ncepbufr ) add_dependencies(python_docs doc python_mod) endif() -install(DIRECTORY ${_lib_dir} DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/python${_PYVER}) -install(PROGRAMS ${_bin_root}/prepbufr2nc DESTINATION ${CMAKE_INSTALL_FULL_BINDIR}) - # Add tests -add_test(NAME test_pyncepbufr_checkpoint - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/test_checkpoint.py - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test/testfiles) -list(APPEND _python_tests test_pyncepbufr_checkpoint) - -add_test(NAME test_pyncepbufr_gps - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/test_gps.py - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test/testfiles) -list(APPEND _python_tests test_pyncepbufr_gps) - -add_test(NAME test_pyncepbufr_prepbufr - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/test_prepbufr.py - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test/testfiles) -list(APPEND _python_tests test_pyncepbufr_prepbufr) - -add_test(NAME test_pyncepbufr_rad - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/test_rad.py - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test/testfiles) -list(APPEND _python_tests test_pyncepbufr_rad) - -add_test(NAME test_pyncepbufr_satwnd - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/test_satwnd.py - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test/testfiles) -list(APPEND _python_tests test_pyncepbufr_satwnd) - -add_test(NAME test_pyncepbufr_write - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/test_write.py - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test/testfiles) -list(APPEND _python_tests test_pyncepbufr_write) - -add_test(NAME test_pyncepbufr_test - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/test.py - WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test/testfiles) -list(APPEND _python_tests test_pyncepbufr_test) - -foreach(_test ${_python_tests}) - set_tests_properties(${_test} - PROPERTIES ENVIRONMENT "PYTHONPATH=${_lib_dir}:$ENV{PYTHONPATH}") +set(pytest_list checkpoint gps misc prepbufr rad satwnd write) +foreach(pytestname ${pytest_list}) + add_test( + NAME test_pyncepbufr_${pytestname} + COMMAND + ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}:$ENV{PYTHONPATH} + ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test/test_${pytestname}.py + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test/testfiles + ) endforeach() diff --git a/python/setup.py.in b/python/setup.py.in deleted file mode 100644 index a28831ef..00000000 --- a/python/setup.py.in +++ /dev/null @@ -1,29 +0,0 @@ -from numpy.distutils.core import setup, Extension - -add_attribute = lambda **args: [list.append(attdict[key], value) - for key, value in args.items()] - -attdict = dict(sources=[], - include_dirs=[], - library_dirs=[], - runtime_library_dirs=[], - libraries=[], - extra_compile_args=[], - extra_objects=[]) - -add_attribute(sources="@CMAKE_CURRENT_SOURCE_DIR@/_bufrlib.pyf") -add_attribute(include_dirs="@CMAKE_BINARY_DIR@/src/include_4") -add_attribute(library_dirs="@CMAKE_BINARY_DIR@/src") -add_attribute(libraries="bufr_4") - -setup(name = "py-ncepbufr", - version = "@PROJECT_VERSION@", - description = "Python interface to NCEP BUFR library", - author = "Jeff Whitaker", - author_email = "jeffrey.s.whitaker@noaa.gov", - license = "GNU Lesser General Public License v3.0", - url = "http://github.com/noaa-emc/nceplibs-bufr", - download_url = "https://github.com/NOAA-EMC/NCEPLIBS-bufr/releases", - ext_modules = [Extension("_bufrlib", **attdict)], - packages = ["ncepbufr"], - scripts = ["utils/prepbufr2nc"]) diff --git a/python/test/test.py b/python/test/test_misc.py similarity index 100% rename from python/test/test.py rename to python/test/test_misc.py From 4f3499e3cb0cc5f782e24499c0fda078d02097f5 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Mon, 13 Nov 2023 17:52:41 -0800 Subject: [PATCH 2/2] remove unneeded python installs in CI --- .github/workflows/Linux.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index cdbd9988..4b6907ac 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -25,8 +25,7 @@ jobs: run: | sudo apt-get update sudo apt-get install python3-pip python3-dev python3-numpy - sudo python3 -m pip install -U pip setuptools - sudo python3 -m pip install -U numpy + sudo python3 -m pip install -U pip sudo python3 -m pip install -U netCDF4 sudo python3 -m pip install -U pdoc