From 38bfbb39d50aa88d6b81c089f2bd39bc27dec7d4 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Thu, 11 Nov 2021 04:46:10 +0000 Subject: [PATCH] fix: buffer over bug #113 Fixed buffer overflow error in MEMCHEK macro (snprintf). This was partly caused by changes in https://github.com/ssandrews/Smoldyn/commit/44ce8fd3fbd5ce7fd1ecdfa9dfc0587fabf9b013 Updated pybind11 to version 2.8.1 chore: using shared_ptr to hold simptr. Use simfree as custom deleter. --- CMakeLists.txt | 15 +- scripts/update_pybind11.sh | 2 +- source/SmolEmulate/SmolEmulate.c | 2 +- source/Smoldyn/smolbng.c | 7 +- source/Smoldyn/smolcmd.c | 3 +- source/Smoldyn/smoldyn.cpp | 7 +- source/Smoldyn/smoldyn.h | 2 +- source/Smoldyn/smolsim.cpp | 4 +- source/libSteve/SimCommand.c | 3 +- source/libSteve/SurfaceParam.c | 32 ++- source/libSteve/parse.c | 12 +- source/libSteve/rxnparam.c | 20 +- source/pybind11/.github/workflows/ci.yml | 87 +++++--- source/pybind11/.github/workflows/pip.yml | 9 +- source/pybind11/.pre-commit-config.yaml | 8 +- source/pybind11/CMakeLists.txt | 6 +- source/pybind11/docs/advanced/exceptions.rst | 4 +- source/pybind11/docs/changelog.rst | 38 ++++ source/pybind11/docs/release.rst | 22 +- .../pybind11/include/pybind11/detail/common.h | 7 +- .../pybind11/detail/type_caster_base.h | 11 +- source/pybind11/include/pybind11/functional.h | 2 +- source/pybind11/include/pybind11/numpy.h | 2 +- source/pybind11/include/pybind11/pybind11.h | 25 +++ source/pybind11/include/pybind11/stl.h | 16 +- source/pybind11/noxfile.py | 5 +- source/pybind11/pybind11/_version.py | 2 +- source/pybind11/tests/CMakeLists.txt | 28 ++- source/pybind11/tests/requirements.txt | 16 +- source/pybind11/tests/test_stl.cpp | 195 +++++++++++++++++- source/pybind11/tests/test_stl.py | 86 ++++++++ .../pybind11/tests/test_virtual_functions.cpp | 40 ++++ .../pybind11/tests/test_virtual_functions.py | 33 +++ source/pybind11/tools/pybind11Common.cmake | 1 + source/pybind11/tools/pybind11NewTools.cmake | 10 +- source/pybind11/tools/pybind11Tools.cmake | 7 +- source/python/Command.h | 6 +- source/python/Simulation.cpp | 76 ++++--- source/python/Simulation.h | 8 +- source/python/module.cpp | 19 -- source/python/util.h | 22 +- 41 files changed, 692 insertions(+), 208 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6486558b..3735e438 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -101,7 +101,6 @@ message(STATUS "Option to build documentation: ${OPTION_DOCS}") ####### Compiler flags ###################### # -# Must use a C++14 compatible compiler (gcc>=5.x). # NOTE: C++17 has parallel implmenetation of many STL algorithms. It would be # nice to use them. Most compilers support C++17 these days. # @@ -153,21 +152,9 @@ if(OPTION_WARNING_AS_ERROR) CheckCompilerFlagAndAdd(-Wno-attributes) endif() -# We need a c++14 support to build python bindings. +# We need at least c++14 support to build python bindings. if(OPTION_PYTHON) set(CMAKE_CXX_STANDARD 14) - try_compile(COMPILER_SUPPORT_C14 - ${CMAKE_BINARY_DIR}/_test_compiler - ${CMAKE_CURRENT_SOURCE_DIR}/cmake/test_compiler.cpp - OUTPUT_VARIABLE COMPILER_TEST_OUTPUT - ) - if(NOT COMPILER_SUPPORT_C14) - message(STATUS "Failed to build test program: ${COMPILER_TEST_OUTPUT}") - message(FATAL_ERROR "Your compiler does not support C++14. " - "Please use a C++14 compliant compiler. " - "See https://en.cppreference.com/w/cpp/compiler_support/14 " - "for C++14 support among C++ compilers.") - endif() endif() diff --git a/scripts/update_pybind11.sh b/scripts/update_pybind11.sh index 21036aea..63b01241 100755 --- a/scripts/update_pybind11.sh +++ b/scripts/update_pybind11.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash git subtree pull --prefix source/pybind11 \ - https://github.com/pybind/pybind11 master --squash + https://github.com/pybind/pybind11 v2.8 --squash diff --git a/source/SmolEmulate/SmolEmulate.c b/source/SmolEmulate/SmolEmulate.c index caf5bb63..1d06bdc6 100644 --- a/source/SmolEmulate/SmolEmulate.c +++ b/source/SmolEmulate/SmolEmulate.c @@ -19,7 +19,7 @@ #define CHECKS(A,...) if(!(A)) {ErrorType=2;sprintf(ErrorString,__VA_ARGS__); goto failure;} else (void)0 -char ErrorString[256]=""; +char ErrorString[STRCHARLONG+STRCHAR]=""; int ErrorType=0; FILE* gnu=NULL; diff --git a/source/Smoldyn/smolbng.c b/source/Smoldyn/smolbng.c index 0aa5c796..4d8b12c4 100644 --- a/source/Smoldyn/smolbng.c +++ b/source/Smoldyn/smolbng.c @@ -1330,14 +1330,17 @@ int bngrunBNGL2(bngptr bng,char *filename,char *outname) { remove(outname); // delete output file snprintf(string,STRCHAR,"perl -v > %s",outname); // test for perl - system(string); + + int n = system(string); + UNUSED(n); + fptr=fopen(outname,"r"); if(!fptr) return 4; remove(outname); snprintf(string,STRCHAR,"perl %s %s %s",bng->bngss->BNG2path,filename,vflag?"":DEVNULL); simLog(bng->bngss->sim,2," Running BNG2.pl on %s\n",filename); - system(string); // generate network + n = system(string); // generate network fptr=fopen(outname,"r"); // check for output file if(!fptr) return 3; // output file was not written diff --git a/source/Smoldyn/smolcmd.c b/source/Smoldyn/smolcmd.c index 96cc5061..21c34f60 100644 --- a/source/Smoldyn/smolcmd.c +++ b/source/Smoldyn/smolcmd.c @@ -329,7 +329,8 @@ enum CMDcode cmdpause(simptr sim,cmdptr cmd,char *line2) { if(line2 && !strcmp(line2,"cmdtype")) return CMDcontrol; if(!sim->graphss || sim->graphss->graphics==0) { fprintf(stderr,"Simulation paused at time %g. Press enter to continue.",sim->time); - scanf("%c",&c); + int n = scanf("%c",&c); + UNUSED(n); return CMDok; } tflag=strchr(sim->flags,'t')?1:0; SCMDCHECK(sim->graphss && sim->graphss->graphics!=0 && !tflag,"pause doesn't work without graphics"); diff --git a/source/Smoldyn/smoldyn.cpp b/source/Smoldyn/smoldyn.cpp index 1e22b534..b6bfc868 100644 --- a/source/Smoldyn/smoldyn.cpp +++ b/source/Smoldyn/smoldyn.cpp @@ -42,10 +42,11 @@ int main(int argc,char **argv) { if(argc<=1) { fprintf(stderr,"Welcome to Smoldyn version %s.\n\n",VERSION); fprintf(stderr,"Enter name of configuration file: "); - fgets(root,STRCHAR,stdin); + char* _x=fgets(root,STRCHAR,stdin); if(strchr(root,'\n')) *(strchr(root,'\n'))='\0'; fprintf(stderr,"Enter runtime flags (q=quiet, p=parameters only), or '-'=none: "); - fgets(flags,STRCHAR,stdin); + _x=fgets(flags,STRCHAR,stdin); + UNUSED(_x); if(strchr(flags,'\n')) *(strchr(flags,'\n'))='\0'; } if(argc>1) { strncpy(root,argv[1],STRCHAR-1); @@ -109,7 +110,7 @@ int main(int argc,char **argv) { smolsimulategl(sim); }} simfree(sim); simfuncfree(); } - + catch (const char* errmsg) { fprintf(stderr, "%s\n", errmsg); exitCode = 1; } diff --git a/source/Smoldyn/smoldyn.h b/source/Smoldyn/smoldyn.h index e31f9a7a..9827b712 100644 --- a/source/Smoldyn/smoldyn.h +++ b/source/Smoldyn/smoldyn.h @@ -77,7 +77,7 @@ enum StructCond #ifndef __string2_h #define __string2_h -#define STRCHAR 256 +#define STRCHAR 512 #define STRCHARLONG 4000 #endif diff --git a/source/Smoldyn/smolsim.cpp b/source/Smoldyn/smolsim.cpp index 5f6993e4..0c59a857 100644 --- a/source/Smoldyn/smolsim.cpp +++ b/source/Smoldyn/smolsim.cpp @@ -50,8 +50,8 @@ FILE *LogFile=NULL; // has to be bigger than ErrorString else compiler emits warning (for a good // reason). // -char ErrorLineAndString[STRCHARLONG]=""; -char ErrorString[STRCHARLONG-STRCHAR-100]=""; +char ErrorLineAndString[STRCHARLONG+STRCHAR+100]=""; +char ErrorString[STRCHARLONG]=""; int ErrorType=0; char SimFlags[STRCHAR]=""; diff --git a/source/libSteve/SimCommand.c b/source/libSteve/SimCommand.c index 5c0eafdf..659ee0c2 100644 --- a/source/libSteve/SimCommand.c +++ b/source/libSteve/SimCommand.c @@ -856,7 +856,8 @@ int scmdopenfiles(cmdssptr cmds,int overwrite) { // When compiled for c++ binary. char str2[STRCHAR]; fprintf(stderr,"Overwrite existing output file '%s' (y/n)? ",cmds->fname[fid]); - scanf("%s",str2); + int count = scanf("%s",str2); + (void)count; if(!(str2[0]=='y' || str2[0]=='Y')) return 1; #endif }} diff --git a/source/libSteve/SurfaceParam.c b/source/libSteve/SurfaceParam.c index dbc2cc4f..8cc7f6ee 100644 --- a/source/libSteve/SurfaceParam.c +++ b/source/libSteve/SurfaceParam.c @@ -13,6 +13,12 @@ files are unavailable */ #include "math2.h" #include "random2.h" +// +// A macro to mark a variable unused. Supresses compiler warnings. +// +#define UNUSED(x) (void)(x) + + /* Declarations for functions that are only used internally */ double interpolate1D(double *xdata,double *ydata,int n,double x); @@ -609,23 +615,26 @@ void xdfmaketableirrev(void) { double *x,*xdfa,*xdfd,xlo,xhi,dx,flux1,flux2,slope1,slope2,intercept1,intercept2,probon,eps,xfitlo,xfithi; int i,i2,n,ifitlo,ifithi,npon; char ynmro[256],ynxdf[256]; + int count=0; fprintf(stderr,"Enter the number of position points for the concentration (e.g. 200): "); - scanf("%i",&n); + count=scanf("%i",&n); if(iseven(n)) n++; fprintf(stderr,"Enter low and high x values (e.g. -6 and 10): "); - scanf("%lf %lf",&xlo,&xhi); + count=scanf("%lf %lf",&xlo,&xhi); fprintf(stderr,"Enter fit domain for x values (e.g. 3 and 7): "); - scanf("%lf %lf",&xfitlo,&xfithi); + count=scanf("%lf %lf",&xfitlo,&xfithi); fprintf(stderr,"Enter epsilon (e.g. 0.0001): "); - scanf("%lf",&eps); + count=scanf("%lf",&eps); fprintf(stderr,"Do you want machine readable output (y/n)? "); - scanf("%s",ynmro); + count=scanf("%s",ynmro); if(ynmro[0]!='y') { fprintf(stderr,"Do you want xdf output (y/n)? "); - scanf("%s",ynxdf); } + count=scanf("%s",ynxdf); } else ynxdf[0]='n'; + UNUSED(count); + x=(double*)calloc(n,sizeof(double)); xdfa=(double*)calloc(n,sizeof(double)); xdfd=(double*)calloc(n,sizeof(double)); @@ -683,18 +692,21 @@ void xdfmaketable(void) { double eps,pon,kon,koff,poff,cs,*x,*xdfa,*xdfd,dx; int i,i2,n,npon,npoff; char yn[256]; + int count=0; fprintf(stderr,"\nFunction for calculating steady-state surface concentrations\n"); fprintf(stderr,"for various adsorption and desorption probabilities.\n\n"); fprintf(stderr,"Enter the number of position points for the concentration (e.g. 200): "); - scanf("%i",&n); + count=scanf("%i",&n); if(n<10) {fprintf(stderr,"Value is too low. Function stopped.\n");return; } if(iseven(n)) n++; fprintf(stderr,"Enter level of precision (e.g. 1e-4): "); - scanf("%lf",&eps); + count=scanf("%lf",&eps); if(eps<=0) {fprintf(stderr,"Impossible precision. Function stopped.\n");return; } fprintf(stderr,"Do you want machine readable output (y/n)? "); - scanf("%s",yn); + count=scanf("%s",yn); + + UNUSED(count); x=(double*)calloc(n,sizeof(double)); xdfa=(double*)calloc(n,sizeof(double)); @@ -848,7 +860,7 @@ void xdfmaketable(void) { double experfcD(double x) { double ans,xxinv; - + if(fabs(x)<20) ans=exp(x*x)*erfccD(x); else { xxinv=1.0/(x*x); diff --git a/source/libSteve/parse.c b/source/libSteve/parse.c index 5b00c1e9..bc67bfbe 100644 --- a/source/libSteve/parse.c +++ b/source/libSteve/parse.c @@ -250,13 +250,13 @@ int Parse_CmdLineArg(int *argcptr,char **argv,ParseFilePtr pfp) { replist=NULL; ndefine=0; maxdefine=0; } - + if(argcptr && *argcptr>0 && argv) { // there are arguments for reading argc=*argcptr; argc2=argc/2; if(!pfp && maxdefine-ndefinedefkey,pfp->ndef,str1)!=-1) pfp->inifdef=1; } - + else if(!strcmp(word,"display_define")) { // display_define Parse_DisplayDefine(pfp); } - + else if(!strcmp(word,"else")) { // else pfp->inifdef=1; } diff --git a/source/libSteve/rxnparam.c b/source/libSteve/rxnparam.c index 03745a82..603a96cf 100644 --- a/source/libSteve/rxnparam.c +++ b/source/libSteve/rxnparam.c @@ -923,6 +923,7 @@ void rdfmaketable() { const double blob=0,bhib=1.0,bincb=0.1; // b value low, high, increment for b -DPYBIND11_FINDPYTHON=ON - - # These items will be removed from the build matrix, keys must match. - exclude: - # Currently 32bit only, and we build 64bit - - runs-on: windows-latest - python: pypy2 - - runs-on: windows-latest - python: pypy3 - - # TODO: PyPy2 7.3.3 segfaults, while 7.3.2 was fine. - - runs-on: ubuntu-latest - python: pypy2 + - runs-on: macos-latest + python: pypy-2.7 name: "🐍 ${{ matrix.python }} • ${{ matrix.runs-on }} • x64 ${{ matrix.args }}" runs-on: ${{ matrix.runs-on }} @@ -93,7 +87,8 @@ jobs: key: ${{ runner.os }}-pip-${{ matrix.python }}-x64-${{ hashFiles('tests/requirements.txt') }} - name: Prepare env - run: python -m pip install -r tests/requirements.txt --prefer-binary + run: | + python -m pip install -r tests/requirements.txt - name: Setup annotations on Linux if: runner.os == 'Linux' @@ -117,7 +112,11 @@ jobs: - name: C++11 tests # TODO: Figure out how to load the DLL on Python 3.8+ +<<<<<<< HEAD if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10-dev'))" +======= + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))" +>>>>>>> 4d545d9c1e4a82ae0d41af44a41a2220111ca47f run: cmake --build . --target cpptest -j 2 - name: Interface test C++11 @@ -145,7 +144,11 @@ jobs: - name: C++ tests # TODO: Figure out how to load the DLL on Python 3.8+ +<<<<<<< HEAD if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10-dev'))" +======= + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))" +>>>>>>> 4d545d9c1e4a82ae0d41af44a41a2220111ca47f run: cmake --build build2 --target cpptest # Third build - C++17 mode with unstable ABI @@ -196,11 +199,20 @@ jobs: fail-fast: false matrix: include: +<<<<<<< HEAD - python-version: 3.9 python-debug: true valgrind: true - python-version: 3.10-dev python-debug: false +======= + # TODO: Fails on 3.10, investigate + - python-version: "3.9" + python-debug: true + valgrind: true + # - python-version: "3.11-dev" + # python-debug: false +>>>>>>> 4d545d9c1e4a82ae0d41af44a41a2220111ca47f name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' • Valgrind' || '' }} • x64" runs-on: ubuntu-latest @@ -244,7 +256,12 @@ jobs: sudo apt-get install libc6-dbg # Needed by Valgrind - name: Prepare env +<<<<<<< HEAD run: python -m pip install -r tests/requirements.txt --prefer-binary +======= + run: | + python -m pip install -r tests/requirements.txt +>>>>>>> 4d545d9c1e4a82ae0d41af44a41a2220111ca47f - name: Configure run: > @@ -521,7 +538,11 @@ jobs: - name: Install dependencies run: | set +e; source /opt/intel/oneapi/setvars.sh; set -e +<<<<<<< HEAD python3 -m pip install -r tests/requirements.txt --prefer-binary +======= + python3 -m pip install -r tests/requirements.txt +>>>>>>> 4d545d9c1e4a82ae0d41af44a41a2220111ca47f - name: Configure C++11 run: | @@ -612,12 +633,20 @@ jobs: run: python3 -m pip install --upgrade pip - name: Install dependencies - run: python3 -m pip install cmake -r tests/requirements.txt --prefer-binary + run: | + python3 -m pip install cmake -r tests/requirements.txt - name: VAR_BUILD_TYPE 7 if: matrix.centos == 7 run: echo MinSizeRel > VAR_BUILD_TYPE +<<<<<<< HEAD + - name: VAR_BUILD_TYPE 7 + if: matrix.centos == 7 + run: echo MinSizeRel > VAR_BUILD_TYPE + +======= +>>>>>>> 4d545d9c1e4a82ae0d41af44a41a2220111ca47f # Using Release to avoid segfault that appeared around 2021-06-04, # apparently when the gcc version changed from 8.3 to 8.4. - name: VAR_BUILD_TYPE 8 @@ -739,8 +768,7 @@ jobs: - 3.7 - 3.8 - 3.9 - - pypy3 - # TODO: fix hang on pypy2 + - pypy-3.6 include: - python: 3.9 @@ -769,7 +797,8 @@ jobs: arch: x86 - name: Prepare env - run: python -m pip install -r tests/requirements.txt --prefer-binary + run: | + python -m pip install -r tests/requirements.txt # First build - C++11 mode and inplace - name: Configure ${{ matrix.args }} @@ -815,7 +844,8 @@ jobs: toolset: 14.0 - name: Prepare env - run: python -m pip install -r tests/requirements.txt --prefer-binary + run: | + python -m pip install -r tests/requirements.txt # First build - C++11 mode and inplace - name: Configure @@ -851,6 +881,10 @@ jobs: std: 17 args: > -DCMAKE_CXX_FLAGS="/permissive- /EHsc /GR" + - python: 3.7 + std: 17 + args: > + -DCMAKE_CXX_FLAGS="/permissive- /EHsc /GR" steps: - uses: actions/checkout@v2 @@ -864,7 +898,8 @@ jobs: uses: jwlawson/actions-setup-cmake@v1.11 - name: Prepare env - run: python -m pip install -r tests/requirements.txt --prefer-binary + run: | + python -m pip install -r tests/requirements.txt # First build - C++11 mode and inplace - name: Configure diff --git a/source/pybind11/.github/workflows/pip.yml b/source/pybind11/.github/workflows/pip.yml index 5547a5e2..203f350b 100644 --- a/source/pybind11/.github/workflows/pip.yml +++ b/source/pybind11/.github/workflows/pip.yml @@ -12,6 +12,9 @@ on: types: - published +env: + PIP_ONLY_BINARY: numpy + jobs: # This builds the sdists and wheels and makes sure the files are exactly as # expected. Using Windows and Python 2.7, since that is often the most @@ -29,7 +32,8 @@ jobs: python-version: 2.7 - name: Prepare env - run: python -m pip install -r tests/requirements.txt --prefer-binary + run: | + python -m pip install -r tests/requirements.txt - name: Python Packaging tests run: pytest tests/extra_python_package/ @@ -50,7 +54,8 @@ jobs: python-version: 3.8 - name: Prepare env - run: python -m pip install -r tests/requirements.txt build twine --prefer-binary + run: | + python -m pip install -r tests/requirements.txt build twine - name: Python Packaging tests run: pytest tests/extra_python_package/ diff --git a/source/pybind11/.pre-commit-config.yaml b/source/pybind11/.pre-commit-config.yaml index c6652d83..b3e51790 100644 --- a/source/pybind11/.pre-commit-config.yaml +++ b/source/pybind11/.pre-commit-config.yaml @@ -61,12 +61,6 @@ repos: hooks: - id: remove-tabs -# Autoremoves unused imports -- repo: https://github.com/hadialqattan/pycln - rev: v1.0.3 - hooks: - - id: pycln - - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.9.0 hooks: @@ -88,7 +82,7 @@ repos: exclude: ^(docs/.*|tools/.*)$ - repo: https://github.com/asottile/yesqa - rev: v1.2.3 + rev: v1.3.0 hooks: - id: yesqa additional_dependencies: *flake8_dependencies diff --git a/source/pybind11/CMakeLists.txt b/source/pybind11/CMakeLists.txt index 25cfcec2..2e81869c 100644 --- a/source/pybind11/CMakeLists.txt +++ b/source/pybind11/CMakeLists.txt @@ -7,13 +7,13 @@ cmake_minimum_required(VERSION 3.4) -# The `cmake_minimum_required(VERSION 3.4...3.21)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.4...3.22)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.21) +if(${CMAKE_VERSION} VERSION_LESS 3.22) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.21) + cmake_policy(VERSION 3.22) endif() # Extract project version from source diff --git a/source/pybind11/docs/advanced/exceptions.rst b/source/pybind11/docs/advanced/exceptions.rst index b4825cbc..40f67d7b 100644 --- a/source/pybind11/docs/advanced/exceptions.rst +++ b/source/pybind11/docs/advanced/exceptions.rst @@ -56,7 +56,9 @@ at its exception handler. +--------------------------------------+--------------------------------------+ | :class:`pybind11::buffer_error` | ``BufferError`` | +--------------------------------------+--------------------------------------+ -| :class:`pybind11::import_error` | ``import_error`` | +| :class:`pybind11::import_error` | ``ImportError`` | ++--------------------------------------+--------------------------------------+ +| :class:`pybind11::attribute_error` | ``AttributeError`` | +--------------------------------------+--------------------------------------+ | Any other exception | ``RuntimeError`` | +--------------------------------------+--------------------------------------+ diff --git a/source/pybind11/docs/changelog.rst b/source/pybind11/docs/changelog.rst index ee79a187..bb5457ee 100644 --- a/source/pybind11/docs/changelog.rst +++ b/source/pybind11/docs/changelog.rst @@ -10,11 +10,49 @@ Starting with version 1.8.0, pybind11 releases use a `semantic versioning IN DEVELOPMENT -------------- +v2.8.1 (Oct 27, 2021) +--------------------- + +Changes and additions: + * The simple namespace creation shortcut added in 2.8.0 was deprecated due to usage of CPython internal API, and will be removed soon. Use ``py::module_::import("types").attr("SimpleNamespace")``. `#3374 `_ +* Add C++ Exception type to throw and catch ``AttributeError``. Useful for + defining custom ``__setattr__`` and ``__getattr__`` methods. + `#3387 `_ + +Fixes: + +* Fixed the potential for dangling references when using properties with + ``std::optional`` types. + `#3376 `_ + +* Modernize usage of ``PyCodeObject`` on Python 3.9+ (moving toward support for + Python 3.11a1) + `#3368 `_ + +* A long-standing bug in ``eigen.h`` was fixed (originally PR #3343). The bug + was unmasked by newly added ``static_assert``'s in the Eigen 3.4.0 release. + `#3352 `_ + +* Support multiple raw inclusion of CMake helper files (Conan.io does this for + multi-config generators). + `#3420 `_ + +* Fix harmless warning on upcoming CMake 3.22. + `#3368 `_ + +* Fix 2.8.0 regression with MSVC 2017 + C++17 mode + Python 3. + `#3407 `_ + +* Fix 2.8.0 regression that caused undefined behavior (typically + segfaults) in ``make_key_iterator``/``make_value_iterator`` if dereferencing + the iterator returned a temporary value instead of a reference. + `#3348 `_ + v2.8.0 (Oct 4, 2021) -------------------- diff --git a/source/pybind11/docs/release.rst b/source/pybind11/docs/release.rst index 7fa254ea..b5de60f4 100644 --- a/source/pybind11/docs/release.rst +++ b/source/pybind11/docs/release.rst @@ -27,7 +27,7 @@ To release a new version of pybind11: ``include/pybind11/detail/common.h``. PATCH should be a simple integer. - Update the version HEX just below, as well. - Update ``pybind11/_version.py`` (match above) - - Run ``nox -s tests_packaging``` to ensure this was done correctly. + - Run ``nox -s tests_packaging`` to ensure this was done correctly. - Ensure that all the information in ``setup.cfg`` is up-to-date, like supported Python versions. - Add release date in ``docs/changelog.rst``. @@ -50,13 +50,15 @@ To release a new version of pybind11: - Make a GitHub release (this shows up in the UI, sends new release notifications to users watching releases, and also uploads PyPI packages). (Note: if you do not use an existing tag, this creates a new lightweight tag - for you, so you could skip the above step). - - GUI method: click "Create a new release" on the far right, fill in the tag - name (if you didn't tag above, it will be made here), fill in a release - name like "Version X.Y.Z", and optionally copy-and-paste the changelog into - the description (processed as markdown by Pandoc). Check "pre-release" if - this is a beta/RC. You can get partway there with - ``cat docs/changelog.rst | pandoc -f rst -t gfm``. + for you, so you could skip the above step.) + + - GUI method: Under `releases `_ + click "Draft a new release" on the far right, fill in the tag name + (if you didn't tag above, it will be made here), fill in a release name + like "Version X.Y.Z", and copy-and-paste the markdown-formatted (!) changelog + into the description (usually ``cat docs/changelog.rst | pandoc -f rst -t gfm``). + Check "pre-release" if this is a beta/RC. + - CLI method: with ``gh`` installed, run ``gh release create vX.Y.Z -t "Version X.Y.Z"`` If this is a pre-release, add ``-p``. @@ -65,7 +67,7 @@ To release a new version of pybind11: - Update version macros in ``include/pybind11/detail/common.h`` (set PATCH to ``0.dev1`` and increment MINOR). - Update ``_version.py`` to match - - Run ``nox -s tests_packaging``` to ensure this was done correctly. + - Run ``nox -s tests_packaging`` to ensure this was done correctly. - Add a spot for in-development updates in ``docs/changelog.rst``. - ``git add``, ``git commit``, ``git push`` @@ -73,7 +75,7 @@ If a version branch is updated, remember to set PATCH to ``1.dev1``. If you'd like to bump homebrew, run: -.. code-block:: +.. code-block:: console brew bump-formula-pr --url https://github.com/pybind/pybind11/archive/vX.Y.Z.tar.gz diff --git a/source/pybind11/include/pybind11/detail/common.h b/source/pybind11/include/pybind11/detail/common.h index 9ad305a4..713de94b 100644 --- a/source/pybind11/include/pybind11/detail/common.h +++ b/source/pybind11/include/pybind11/detail/common.h @@ -10,12 +10,12 @@ #pragma once #define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 9 -#define PYBIND11_VERSION_PATCH 0.dev1 +#define PYBIND11_VERSION_MINOR 8 +#define PYBIND11_VERSION_PATCH 1 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020900D1 +#define PYBIND11_VERSION_HEX 0x02080100 #define PYBIND11_NAMESPACE_BEGIN(name) namespace name { #define PYBIND11_NAMESPACE_END(name) } @@ -831,6 +831,7 @@ PYBIND11_RUNTIME_EXCEPTION(value_error, PyExc_ValueError) PYBIND11_RUNTIME_EXCEPTION(type_error, PyExc_TypeError) PYBIND11_RUNTIME_EXCEPTION(buffer_error, PyExc_BufferError) PYBIND11_RUNTIME_EXCEPTION(import_error, PyExc_ImportError) +PYBIND11_RUNTIME_EXCEPTION(attribute_error, PyExc_AttributeError) PYBIND11_RUNTIME_EXCEPTION(cast_error, PyExc_RuntimeError) /// Thrown when pybind11::cast or handle::call fail due to a type casting error PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used internally diff --git a/source/pybind11/include/pybind11/detail/type_caster_base.h b/source/pybind11/include/pybind11/detail/type_caster_base.h index f804d9d1..00ce1a7a 100644 --- a/source/pybind11/include/pybind11/detail/type_caster_base.h +++ b/source/pybind11/include/pybind11/detail/type_caster_base.h @@ -468,12 +468,19 @@ PYBIND11_NOINLINE std::string error_string() { PyFrameObject *frame = trace->tb_frame; errorString += "\n\nAt:\n"; while (frame) { +#if PY_VERSION_HEX >= 0x03090000 + PyCodeObject *f_code = PyFrame_GetCode(frame); +#else + PyCodeObject *f_code = frame->f_code; + Py_INCREF(f_code); +#endif int lineno = PyFrame_GetLineNumber(frame); errorString += - " " + handle(frame->f_code->co_filename).cast() + + " " + handle(f_code->co_filename).cast() + "(" + std::to_string(lineno) + "): " + - handle(frame->f_code->co_name).cast() + "\n"; + handle(f_code->co_name).cast() + "\n"; frame = frame->f_back; + Py_DECREF(f_code); } } #endif diff --git a/source/pybind11/include/pybind11/functional.h b/source/pybind11/include/pybind11/functional.h index 24141ce3..ad5608c2 100644 --- a/source/pybind11/include/pybind11/functional.h +++ b/source/pybind11/include/pybind11/functional.h @@ -69,7 +69,7 @@ struct type_caster> { // ensure GIL is held during functor destruction struct func_handle { function f; -#if !(defined(_MSC_VER) && _MSC_VER == 1916 && defined(PYBIND11_CPP17) && PY_MAJOR_VERSION < 3) +#if !(defined(_MSC_VER) && _MSC_VER == 1916 && defined(PYBIND11_CPP17)) // This triggers a syntax error under very special conditions (very weird indeed). explicit #endif diff --git a/source/pybind11/include/pybind11/numpy.h b/source/pybind11/include/pybind11/numpy.h index b7747fae..b43a7716 100644 --- a/source/pybind11/include/pybind11/numpy.h +++ b/source/pybind11/include/pybind11/numpy.h @@ -518,7 +518,7 @@ class dtype : public object { } /// Single-character for dtype's type. - /// For example, ``float`` is 'f', ``double`` 'd', ``int`` 'i', and ``long`` 'd'. + /// For example, ``float`` is 'f', ``double`` 'd', ``int`` 'i', and ``long`` 'l'. char char_() const { // Note: The signature, `dtype::char_` follows the naming of NumPy's // public Python API (i.e., ``dtype.char``), rather than its internal diff --git a/source/pybind11/include/pybind11/pybind11.h b/source/pybind11/include/pybind11/pybind11.h index 095efd9c..bfc1c368 100644 --- a/source/pybind11/include/pybind11/pybind11.h +++ b/source/pybind11/include/pybind11/pybind11.h @@ -2335,6 +2335,29 @@ inline function get_type_override(const void *this_ptr, const type_info *this_ty /* Don't call dispatch code if invoked from overridden function. Unfortunately this doesn't work on PyPy. */ #if !defined(PYPY_VERSION) + +#if PY_VERSION_HEX >= 0x03090000 + PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get()); + if (frame != nullptr) { + PyCodeObject *f_code = PyFrame_GetCode(frame); + // f_code is guaranteed to not be NULL + if ((std::string) str(f_code->co_name) == name && f_code->co_argcount > 0) { + PyObject* locals = PyEval_GetLocals(); + if (locals != nullptr) { + PyObject *self_caller = dict_getitem( + locals, PyTuple_GET_ITEM(f_code->co_varnames, 0) + ); + if (self_caller == self.ptr()) { + Py_DECREF(f_code); + Py_DECREF(frame); + return function(); + } + } + } + Py_DECREF(f_code); + Py_DECREF(frame); + } +#else PyFrameObject *frame = PyThreadState_Get()->frame; if (frame != nullptr && (std::string) str(frame->f_code->co_name) == name && frame->f_code->co_argcount > 0) { @@ -2344,6 +2367,8 @@ inline function get_type_override(const void *this_ptr, const type_info *this_ty if (self_caller == self.ptr()) return function(); } +#endif + #else /* PyPy currently doesn't provide a detailed cpyext emulation of frame objects, so we have to emulate this using Python. This diff --git a/source/pybind11/include/pybind11/stl.h b/source/pybind11/include/pybind11/stl.h index 2c017b4f..3608d298 100644 --- a/source/pybind11/include/pybind11/stl.h +++ b/source/pybind11/include/pybind11/stl.h @@ -245,17 +245,17 @@ template , Key, Value> { }; // This type caster is intended to be used for std::optional and std::experimental::optional -template struct optional_caster { - using value_conv = make_caster; +template struct optional_caster { + using value_conv = make_caster; - template - static handle cast(T_ &&src, return_value_policy policy, handle parent) { + template + static handle cast(T &&src, return_value_policy policy, handle parent) { if (!src) return none().inc_ref(); if (!std::is_lvalue_reference::value) { - policy = return_value_policy_override::policy(policy); + policy = return_value_policy_override::policy(policy); } - return value_conv::cast(*std::forward(src), policy, parent); + return value_conv::cast(*std::forward(src), policy, parent); } bool load(handle src, bool convert) { @@ -269,11 +269,11 @@ template struct optional_caster { if (!inner_caster.load(src, convert)) return false; - value.emplace(cast_op(std::move(inner_caster))); + value.emplace(cast_op(std::move(inner_caster))); return true; } - PYBIND11_TYPE_CASTER(T, _("Optional[") + value_conv::name + _("]")); + PYBIND11_TYPE_CASTER(Type, _("Optional[") + value_conv::name + _("]")); }; #if defined(PYBIND11_HAS_OPTIONAL) diff --git a/source/pybind11/noxfile.py b/source/pybind11/noxfile.py index 23417982..757a5384 100644 --- a/source/pybind11/noxfile.py +++ b/source/pybind11/noxfile.py @@ -2,7 +2,7 @@ nox.options.sessions = ["lint", "tests", "tests_packaging"] -PYTHON_VERISONS = ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10"] +PYTHON_VERISONS = ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] @nox.session(reuse_venv=True) @@ -20,7 +20,8 @@ def tests(session: nox.Session) -> None: Run the tests (requires a compiler). """ tmpdir = session.create_tmp() - session.install("pytest", "cmake") + session.install("cmake") + session.install("-r", "tests/requirements.txt") session.run( "cmake", "-S", diff --git a/source/pybind11/pybind11/_version.py b/source/pybind11/pybind11/_version.py index ce894a77..7cc10028 100644 --- a/source/pybind11/pybind11/_version.py +++ b/source/pybind11/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s): return s -__version__ = "2.9.0.dev1" +__version__ = "2.8.1" version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/source/pybind11/tests/CMakeLists.txt b/source/pybind11/tests/CMakeLists.txt index d4e7b71e..6dc67a11 100644 --- a/source/pybind11/tests/CMakeLists.txt +++ b/source/pybind11/tests/CMakeLists.txt @@ -174,10 +174,14 @@ set(PYBIND11_CROSS_MODULE_GIL_TESTS test_gil_scoped.py) set(PYBIND11_EIGEN_REPO "https://gitlab.com/libeigen/eigen.git" CACHE STRING "Eigen repository to use for tests") -# This hash is for 3.4.0, using a hash for security reasons -set(PYBIND11_EIGEN_VERSION - "929bc0e191d0927b1735b9a1ddc0e8b77e3a25ec" - CACHE STRING "Eigen version to use for tests") +# Always use a hash for reconfigure speed and security reasons +# Include the version number for pretty printing (keep in sync) +set(PYBIND11_EIGEN_VERSION_AND_HASH + "3.4.0;929bc0e191d0927b1735b9a1ddc0e8b77e3a25ec" + CACHE STRING "Eigen version to use for tests, format: VERSION;HASH") + +list(GET PYBIND11_EIGEN_VERSION_AND_HASH 0 PYBIND11_EIGEN_VERSION_STRING) +list(GET PYBIND11_EIGEN_VERSION_AND_HASH 1 PYBIND11_EIGEN_VERSION_HASH) # Check if Eigen is available; if not, remove from PYBIND11_TEST_FILES (but # keep it in PYBIND11_PYTEST_FILES, so that we get the "eigen is not installed" @@ -196,16 +200,22 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) FetchContent_Declare( eigen GIT_REPOSITORY "${PYBIND11_EIGEN_REPO}" - GIT_TAG "${PYBIND11_EIGEN_VERSION}") + GIT_TAG "${PYBIND11_EIGEN_VERSION_HASH}") FetchContent_GetProperties(eigen) if(NOT eigen_POPULATED) - message(STATUS "Downloading Eigen") + message( + STATUS + "Downloading Eigen ${PYBIND11_EIGEN_VERSION_STRING} (${PYBIND11_EIGEN_VERSION_HASH}) from ${PYBIND11_EIGEN_REPO}" + ) FetchContent_Populate(eigen) endif() set(EIGEN3_INCLUDE_DIR ${eigen_SOURCE_DIR}) set(EIGEN3_FOUND TRUE) + # When getting locally, the version is not visible from a superprojet, + # so just force it. + set(EIGEN3_VERSION "${PYBIND11_EIGEN_VERSION_STRING}") else() find_package(Eigen3 3.2.7 QUIET CONFIG) @@ -256,7 +266,9 @@ if(Boost_FOUND) endif() # Check if we need to add -lstdc++fs or -lc++fs or nothing -if(MSVC) +if(DEFINED CMAKE_CXX_STANDARD AND CMAKE_CXX_STANDARD LESS 17) + set(STD_FS_NO_LIB_NEEDED TRUE) +elseif(MSVC) set(STD_FS_NO_LIB_NEEDED TRUE) else() file( @@ -286,7 +298,7 @@ elseif(${STD_FS_NEEDS_CXXFS}) elseif(${STD_FS_NO_LIB_NEEDED}) set(STD_FS_LIB "") else() - message(WARNING "Unknown compiler - not passing -lstdc++fs") + message(WARNING "Unknown C++17 compiler - not passing -lstdc++fs") set(STD_FS_LIB "") endif() diff --git a/source/pybind11/tests/requirements.txt b/source/pybind11/tests/requirements.txt index 069122b8..8d2742a7 100644 --- a/source/pybind11/tests/requirements.txt +++ b/source/pybind11/tests/requirements.txt @@ -1,12 +1,12 @@ ---extra-index-url https://antocuni.github.io/pypy-wheels/manylinux2010/ -numpy==1.16.6; python_version<"3.6" and sys_platform!="win32" -numpy==1.18.0; platform_python_implementation=="PyPy" and sys_platform=="darwin" and python_version>="3.6" -numpy==1.19.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version=="3.6" -numpy==1.21.2; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.7" and python_version<"3.10" -numpy==1.21.2; platform_python_implementation!="PyPy" and sys_platform=="linux" and python_version=="3.10" +numpy==1.16.6; python_version<"3.6" and sys_platform!="win32" and platform_python_implementation!="PyPy" +numpy==1.19.0; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.6" +numpy==1.20.0; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.7" +numpy==1.19.3; platform_python_implementation!="PyPy" and python_version=="3.6" +numpy==1.21.3; platform_python_implementation!="PyPy" and python_version>="3.7" +py @ git+https://github.com/pytest-dev/py; python_version>="3.11" pytest==4.6.9; python_version<"3.5" pytest==6.1.2; python_version=="3.5" pytest==6.2.4; python_version>="3.6" pytest-timeout -scipy==1.2.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version<"3.6" -scipy==1.5.4; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.6" and python_version<"3.10" +scipy==1.2.3; platform_python_implementation!="PyPy" and python_version<"3.6" +scipy==1.5.4; platform_python_implementation!="PyPy" and python_version>="3.6" and python_version<"3.10" diff --git a/source/pybind11/tests/test_stl.cpp b/source/pybind11/tests/test_stl.cpp index 7e3363c5..70c76916 100644 --- a/source/pybind11/tests/test_stl.cpp +++ b/source/pybind11/tests/test_stl.cpp @@ -19,6 +19,18 @@ #include #include +#if defined(PYBIND11_TEST_BOOST) +#include + +namespace pybind11 { namespace detail { +template +struct type_caster> : optional_caster> {}; + +template <> +struct type_caster : void_caster {}; +}} // namespace pybind11::detail +#endif + // Test with `std::variant` in C++17 mode, or with `boost::variant` in C++11/14 #if defined(PYBIND11_HAS_VARIANT) using std::variant; @@ -59,7 +71,8 @@ namespace std { template