Support for building NEURON on windows under Azure CI with installer

azure-pipelines.yml

@@ -22,6 +22,7 @@ trigger:
 jobs:
 
 - job: 'ManyLinuxWheels'
+  timeoutInMinutes: 30
   pool:
     vmImage: 'ubuntu-18.04'
   strategy:
@@ -57,6 +58,7 @@ jobs:
 
 # Jobs to build OSX wheels natively
 - job: 'MacOSWheels'
+  timeoutInMinutes: 30
   pool:
     vmImage: 'macOS-10.15'
   strategy:
@@ -101,3 +103,33 @@ jobs:
     displayName: 'Build MacOS Wheel'
 
   - template: ci/azure-wheel-test-upload.yml
+
+
+# Jobs to build NEURON installer natively
+- job: 'WindowsInstaller'
+  timeoutInMinutes: 45
+  pool:
+    vmImage: windows-2019
+  variables:
+    MSYS2_ROOT: C:\msys64
+
+  steps:
+
+  - task: BatchScript@1
+    inputs:
+      filename: ci/win_download_deps.cmd
+    displayName: "Download Dependencies"
+    condition: succeeded()
+
+  - task: BatchScript@1
+    inputs:
+      filename: ci/win_install_deps.cmd
+    displayName: "Install Dependencies"
+    condition: succeeded()
+
+  - script: |
+      %MSYS2_ROOT%\usr\bin\bash -lc "$BUILD_SOURCESDIRECTORY/ci/win_build_cmake.sh"
+    displayName: "Build and Create Installer"
+    condition: succeeded()
+
+  - template: ci/azure-win-installer-upload.yml

ci/azure-win-installer-upload.yml

@@ -0,0 +1,37 @@
+steps:
+
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: '2.7'
+    displayName: "Use System Python"
+
+# Note: silently installing neuron doesn't work, when we print out the contents of the installation folder not all files
+# are there. We suspect an Azure issue since we've tried a lot of things (to name a few: trying NSIS 3.04 commit that
+# was working back then, changing NSIS installers, trying latest NSIS, different process waiting strategies including
+# powershell and start /wait and so on) and locally testing the installer works.
+  - task: BatchScript@1
+    inputs:
+      filename: ci/win_test_installer.cmd
+    displayName: "Test Installer"
+    condition: succeeded()
+
+  - task: GithubRelease@0
+    inputs:
+      gitHubConnection: neuronsimulator-installers
+      repositoryName: neuronsimulator/installers
+      action: edit
+      # note : if previous release doesn't exist, as we are pushing
+      # to different repository, we must need to provide commit id.
+      target: '9d7b1e26717bd207f7ae6114f78f7ab7b958c998'
+      tagSource: manual
+      tag: nightly
+      title: 'Nightly NEURON Developer Snapshot'
+      releaseNotesSource: input
+      releaseNotes: "Last NEURON Commit: $(Build.SourceVersionMessage)"
+      isPreRelease: true
+      assetUploadMode: replace
+      addChangeLog: false
+      assets: |
+        $(Build.SourcesDirectory)\nrn-nightly-AMD64.exe
+    displayName: 'Upload installer to GitHub'
+    condition: and(succeeded(), in(variables['Build.Reason'], 'Manual', 'Schedule'))

ci/win_build_cmake.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -ex
+
+# standard env variables from ming2 shell
+export MSYSTEM_CHOST=x86_64-w64-mingw32
+export MSYSTEM=MINGW64
+export MINGW_PREFIX=/mingw64
+export MINGW_CHOST=x86_64-w64-mingw32
+export MSYSTEM_PREFIX=/mingw64
+export PATH=/mingw64/bin:$PATH
+
+mkdir -p $BUILD_SOURCESDIRECTORY/build
+cd $BUILD_SOURCESDIRECTORY/build
+
+# build and create installer
+/mingw64/bin/cmake .. \
+	-G 'Unix Makefiles'  \
+	-DNRN_ENABLE_MPI_DYNAMIC=ON  \
+	-DNRN_ENABLE_MPI=ON  \
+	-DCMAKE_PREFIX_PATH='/c/msmpi'  \
+	-DNRN_ENABLE_INTERVIEWS=ON  \
+	-DNRN_ENABLE_PYTHON=ON  \
+	-DNRN_ENABLE_RX3D=ON  \
+	-DNRN_RX3D_OPT_LEVEL=2 \
+	-DPYTHON_EXECUTABLE=/c/Python35/python.exe \
+	-DNRN_ENABLE_PYTHON_DYNAMIC=ON  \
+	-DNRN_PYTHON_DYNAMIC='c:/Python35/python.exe;c:/Python36/python.exe;c:/Python37/python.exe;c:/Python38/python.exe;c:/Python27/python.exe'  \
+	-DCMAKE_INSTALL_PREFIX='/c/nrn-install' \
+	-DMPI_CXX_LIB_NAMES:STRING=msmpi \
+	-DMPI_C_LIB_NAMES:STRING=msmpi \
+	-DMPI_msmpi_LIBRARY:FILEPATH=c:/msmpi/lib/x64/msmpi.lib
+make -j
+ctest -VV
+make install
+make setup_exe
+
+# copy installer with fixed name for nightly upload
+cp src/mswin/nrn*AMD64.exe $BUILD_SOURCESDIRECTORY/nrn-nightly-AMD64.exe

ci/win_download_deps.cmd

@@ -0,0 +1,25 @@
+@echo on
+
+:: download all installers
+
+:: python
+pwsh -command Invoke-WebRequest -MaximumRetryCount 4 -OutFile python-3.5.exe https://www.python.org/ftp/python/3.5.4/python-3.5.4-amd64.exe || goto :error
+pwsh -command Invoke-WebRequest -MaximumRetryCount 4 -OutFile python-3.6.exe https://www.python.org/ftp/python/3.6.8/python-3.6.8-amd64.exe || goto :error
+pwsh -command Invoke-WebRequest -MaximumRetryCount 4 -OutFile python-3.7.exe https://www.python.org/ftp/python/3.7.7/python-3.7.7-amd64.exe || goto :error
+pwsh -command Invoke-WebRequest -MaximumRetryCount 4 -OutFile python-3.8.exe https://www.python.org/ftp/python/3.8.2/python-3.8.2-amd64.exe || goto :error
+pwsh -command Invoke-WebRequest -MaximumRetryCount 4 -OutFile python-2.7.msi https://www.python.org/ftp/python/2.7.17/python-2.7.17.amd64.msi || goto :error
+
+:: mpi
+pwsh -command Invoke-WebRequest -MaximumRetryCount 4 -OutFile msmpisetup.exe https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisetup.exe || goto :error
+pwsh -command Invoke-WebRequest -MaximumRetryCount 4 -OutFile msmpisdk.msi https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623/msmpisdk.msi || goto :error
+
+:: nsis plugin
+pwsh -command Invoke-WebRequest -MaximumRetryCount 4 -OutFile EnVar_pugin.zip https://nsis.sourceforge.io/mediawiki/images/7/7f/EnVar_plugin.zip || goto :error
+
+:: if all goes well, go to end
+goto :EOF
+
+:: something has failed, teminate with error code
+:error
+echo ERROR : exiting with error code %errorlevel%
+exit /b %errorlevel%

ci/win_install_deps.cmd

@@ -0,0 +1,75 @@
+@echo on
+
+:: install all dependencies
+
+:: install python
+start /wait msiexec /i python-2.7.msi /norestart /qn TARGETDIR="C:\Python27" ADDLOCAL=ALL || goto :error
+python-3.5.exe /passive Include_pip=1 Include_test=0 PrependPath=1 DefaultJustForMeTargetDir=C:\Python35 || goto :error
+python-3.6.exe /passive Include_pip=1 Include_test=0 PrependPath=1 DefaultJustForMeTargetDir=C:\Python36 || goto :error
+python-3.7.exe /passive Include_pip=1 Include_test=0 PrependPath=1 DefaultJustForMeTargetDir=C:\Python37 || goto :error
+python-3.8.exe /passive Include_pip=1 Include_test=0 PrependPath=1 DefaultJustForMeTargetDir=C:\Python38 || goto :error
+
+:: fix msvcc version for all python3
+pwsh -command "(Get-Content C:\Python35\Lib\distutils\cygwinccompiler.py) -replace 'elif msc_ver == ''1600'':', 'elif msc_ver == ''1900'':' | Out-File C:\Python35\Lib\distutils\cygwinccompiler.py"
+pwsh -command "(Get-Content C:\Python36\Lib\distutils\cygwinccompiler.py) -replace 'elif msc_ver == ''1600'':', 'elif msc_ver == ''1916'':' | Out-File C:\Python36\Lib\distutils\cygwinccompiler.py"
+pwsh -command "(Get-Content C:\Python37\Lib\distutils\cygwinccompiler.py) -replace 'elif msc_ver == ''1600'':', 'elif msc_ver == ''1900'':' | Out-File C:\Python37\Lib\distutils\cygwinccompiler.py"
+pwsh -command "(Get-Content C:\Python38\Lib\distutils\cygwinccompiler.py) -replace 'elif msc_ver == ''1600'':', 'elif msc_ver == ''1916'':' | Out-File C:\Python38\Lib\distutils\cygwinccompiler.py"
+
+:: fix msvc runtime library for all python
+pwsh -command "(Get-Content C:\Python35\Lib\distutils\cygwinccompiler.py) -replace 'msvcr100', 'msvcrt' | Out-File C:\Python35\Lib\distutils\cygwinccompiler.py"
+pwsh -command "(Get-Content C:\Python36\Lib\distutils\cygwinccompiler.py) -replace 'msvcr100', 'msvcrt' | Out-File C:\Python36\Lib\distutils\cygwinccompiler.py"
+pwsh -command "(Get-Content C:\Python37\Lib\distutils\cygwinccompiler.py) -replace 'msvcr100', 'msvcrt' | Out-File C:\Python37\Lib\distutils\cygwinccompiler.py"
+pwsh -command "(Get-Content C:\Python38\Lib\distutils\cygwinccompiler.py) -replace 'msvcr100', 'msvcrt' | Out-File C:\Python38\Lib\distutils\cygwinccompiler.py"
+pwsh -command "(Get-Content C:\Python27\Lib\distutils\cygwinccompiler.py) -replace 'msvcr90', 'msvcrt' | Out-File C:\Python27\Lib\distutils\cygwinccompiler.py"
+
+:: install numpy
+C:\Python35\python.exe -m pip install numpy || goto :error
+C:\Python36\python.exe -m pip install numpy || goto :error
+C:\Python37\python.exe -m pip install numpy || goto :error
+C:\Python38\python.exe -m pip install numpy || goto :error
+C:\Python27\python.exe -m pip install numpy || goto :error
+
+:: install nsis
+choco install nsis --no-progress -y  --version=3.05 --params="/InstallDir:C\Program Files\ (x86)\NSIS /NoUpdate /NoPath" || goto :error
+pwsh -command Expand-Archive EnVar_pugin.zip -DestinationPath "${env:ProgramFiles(x86)}\NSIS" || goto :error
+
+:: install mpi
+msmpisetup.exe -unattend -installroot C:\msmpi || goto :error
+start /wait msiexec /i msmpisdk.msi /norestart /qn INSTALLDIR="C:\msmpi" ADDLOCAL=ALL || goto :error
+rename C:\msmpi\Bin  bin || goto :error
+rename C:\msmpi\Lib  lib || goto :error
+rename C:\msmpi\Include include || goto :error
+copy "c:\Windows\System32\msmpi.dll" "c:\msmpi\lib\x64\msmpi.dll" || goto :error
+copy "c:\Windows\SysWoW64\msmpi.dll" "c:\msmpi\lib\x86\msmpi.dll" || goto :error
+
+:: copy files to necessary directories
+cp 'C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x64\rc.exe' 'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\' || goto :error
+cp 'C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x64\rcdll.dll' 'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\' || goto :error
+
+:: install msys2 / mingw packages. NOTE : do not goto error because msys2 is already installed and might fail
+set PATH=%MSYS2_ROOT%\usr\bin;%SystemRoot%\system32;%SystemRoot%;%SystemRoot%\System32\Wbem;%PATH%
+choco install --no-progress msys2 --params="/InstallDir:%MSYS2_ROOT% /NoUpdate /NoPath" || goto :error
+
+%MSYS2_ROOT%\usr\bin\pacman --noconfirm --needed -S --disable-download-timeout ^
+git ^
+zip ^
+unzip ^
+base-devel ^
+mingw-w64-x86_64-cmake ^
+mingw-w64-x86_64-ncurses ^
+mingw-w64-x86_64-readline ^
+mingw-w64-x86_64-python2 ^
+mingw-w64-x86_64-python3 ^
+mingw64/mingw-w64-x86_64-cython ^
+mingw-w64-x86_64-python3-setuptools ^
+mingw-w64-x86_64-python3-pip ^
+mingw64/mingw-w64-x86_64-dlfcn ^
+mingw-w64-x86_64-toolchain || goto :error
+
+:: if all goes well, go to end
+goto :EOF
+
+:: something has failed, teminate with error code
+:error
+echo ERROR : exiting with error code %errorlevel%
+exit /b %errorlevel%

ci/win_test_installer.cmd

@@ -0,0 +1,79 @@
+@echo on
+
+:: install numpy dependency
+python -m pip install numpy
+
+:: install installer
+:: TODO : need to fix this as next command will not wait till installer finishes
+.\nrn-nightly-AMD64.exe /S /D=C:\nrn_test
+
+:: take a look
+dir C:\nrn_test
+tree /F C:\nrn_test\lib\python
+
+:: setup environment
+set PATH=C:\nrn_test\bin;%PATH%
+set PYTHONPATH=C:\nrn_test\lib\python;%PYTHONPATH%
+set NEURONHOME=C:\nrn_test
+
+:: test all pythons
+C:\Python27\python -c "import neuron; neuron.test(); neuron.test_rxd(); quit()" || goto :error
+C:\Python35\python -c "import neuron; neuron.test(); neuron.test_rxd(); quit()" || goto :error
+C:\Python36\python -c "import neuron; neuron.test(); neuron.test_rxd(); quit()" || goto :error
+C:\Python37\python -c "import neuron; neuron.test(); neuron.test_rxd(); quit()" || goto :error
+C:\Python38\python -c "import neuron; neuron.test(); neuron.test_rxd(); quit()" || goto :error
+
+:: run also using whatever is system python
+python --version
+python -c "import neuron; neuron.test(); neuron.test_rxd(); quit()" || goto :error
+
+:: test python and nrniv
+python -c "from neuron import h; s = h.Section(); s.insert('hh'); quit()" || goto :error
+nrniv -python -c "from neuron import h; s = h.Section(); s.insert('hh'); quit()" || goto :error
+
+:: test mpi
+mpiexec -n 2 nrniv %cd%\src\parallel\test0.hoc -mpi || goto :error
+mpiexec -n 2 python %cd%\src\parallel\test0.py -mpi --expected-hosts 2 || goto :error
+
+:: test of association with hoc files
+del temp.txt
+echo wopen("temp.txt") > .\temp.hoc
+echo fprint("hello\n") >> .\temp.hoc
+echo wopen() >> .\temp.hoc
+echo quit() >> .\temp.hoc
+start .\temp.hoc
+ping -n 10 127.0.0.1
+cat temp.txt
+findstr /i "^hello$" temp.txt || goto :error
+
+:: setup for mknrndll/nrnivmodl
+set N=C:\nrn_test
+set PATH=C:\nrn_test\mingw\usr\bin;%PATH%
+
+:: test mknrndll
+copy /A share\examples\nrniv\nmodl\cacum.mod .
+C:\nrn_test\mingw\usr\bin\bash -c "mknrndll" || goto :error
+python -c "import neuron; from neuron import h; s = h.Section(); s.insert('cacum'); print('cacum inserted'); quit()" || goto :error
+
+:: test nrnivmodl
+rm -f cacum* mod_func* nrnmech.dll
+copy /A share\examples\nrniv\nmodl\cacum.mod .
+call nrnivmodl
+echo "nrnivmodl successfull"
+python -c "import neuron; from neuron import h; s = h.Section(); s.insert('cacum'); print('cacum inserted'); quit()" || goto :error
+
+echo "All tests finished!"
+
+:: uninstall neuron
+C:\nrn_test\uninstall /S || goto :error
+echo "Uninstalled NEURON"
+
+:: if all goes well, go to end
+goto :EOF
+
+:: something has failed, teminate with error code
+:error
+echo ERROR : NOT exiting with error code %errorlevel% foor now ..
+
+:: TODO - fix silent installer and exit from here
+:: exit /b %errorlevel%

cmake/ConfigFileSetting.cmake

@@ -306,7 +306,7 @@ if(MINGW)
   nrn_configure_file(mknrndll.mak src/mswin/lib)
 endif()
 # TODO temporary workaround for mingw
-file(COPY ${PROJECT_BINARY_DIR}/share/nrn/lib/nrnunits.lib DESTINATION ${PROJECT_BINARY_DIR}/lib)
+file(COPY ${PROJECT_BINARY_DIR}/share/nrn/lib/nrnunits.lib.in DESTINATION ${PROJECT_BINARY_DIR}/lib)
 
 # =============================================================================
 # If Interviews is not provided, configure local files

mingw_files/vcenv.sh

@@ -1,21 +1,11 @@
-
 export UCRTContentRoot='C:\Program Files (x86)\Windows Kits\10\'
-export INCLUDE='C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\INCLUDE;C:\Program Files (x86)\Windows Kits\10\include\10.0.10240.0\ucrt;C:\Program Files (x86)\Windows Kits\8.1\include\shared;C:\Program Files (x86)\Windows Kits\8.1\include\um;C:\Program Files (x86)\Windows Kits\8.1\include\winrt;'
-export curDir='C:\Program Files (x86)\Microsoft Visual C++ Build Tools\'
-export FrameworkVersion64='v4.0.30319'
-export WindowsLibPath='References\CommonConfiguration\Neutral'
-export FrameworkDIR64='C:\Windows\Microsoft.NET\Framework64'
+export INCLUDE='C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include;C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt;'
 export VCTargetsPath='C:\Program Files (x86)\MSBuild\Microsoft.Cpp\v4.0\v140\'
 export Platform='X64'
-export CommandPromptType='Native'
-export FrameworkDir='C:\Windows\Microsoft.NET\Framework64'
-export WindowsSDKLibVersion='winv6.3\'
 export VCINSTALLDIR='C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\'
-export LIB='C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\LIB\amd64;C:\Program Files (x86)\Windows Kits\10\lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\8.1\lib\winv6.3\um\x64;'
-export LIBPATH='C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\LIB\amd64;C:\Windows\Microsoft.NET\Framework64\v4.0.30319;C:\Windows\Microsoft.NET\Framework64\;References\CommonConfiguration\Neutral;\Microsoft.VCLibs\14.0\References\CommonConfiguration\neutral;'
-export CurrentDir='C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC'
-export WindowsSdkDir='C:\Program Files (x86)\Windows Kits\8.1\'
+export LIB='C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\lib\amd64;C:\Program Files (x86)\Windows Kits\10\lib\10.0.10240.0\ucrt\x64;'
+export LIBPATH='C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\lib\amd64;'
+export WindowsSdkDir='C:\Program Files (x86)\Windows Kits\10\'
 export UCRTVersion='10.0.10240.0'
-export FrameworkVersion='v4.0.30319'
 export UniversalCRTSdkDir='C:\Program Files (x86)\Windows Kits\10'
-export PATH='/c/Program Files (x86)/MSBuild/14.0/bin/amd64:/c/Program Files (x86)/Microsoft Visual Studio 14.0/VC/BIN/amd64:/c/Windows/Microsoft.NET/Framework64/v4.0.30319:/c/Windows/Microsoft.NET/Framework64:/c/Program Files (x86)/Windows Kits/8.1/bin/x64:/c/Program Files (x86)/Windows Kits/8.1/bin/x86':$PATH
+export PATH='/c/Program Files (x86)/Microsoft Visual Studio 14.0/VC/bin/amd64:/c/Program Files (x86)/Windows Kits/10/bin/x64:/c/Program Files (x86)/Windows Kits/10/bin/x86':$PATH

packaging/python/test_wheels.sh

@@ -69,16 +69,19 @@ run_serial_test () {
     # Test 5: run basic tests via python while loading shared library
     $python_exe -c "import neuron; neuron.test(); neuron.test_rxd(); quit()"
 
-    # Test 6: run basic tests via special : azure pipelines get stuck with their
-    # own python from hosted cache (most likely security settings or bash behaviour).
+    # Test 6: run basic test to use compiled mod file
+    $python_exe -c "import neuron; from neuron import h; s = h.Section(); s.insert('cacum'); quit()"
+
+    # Test 7: run basic tests via special : azure pipelines get stuck with their
+    # own python from hosted cache (most likely security settings).
     if [[ "$SKIP_EMBEDED_PYTHON_TEST" != "true" ]]; then
       ./x86_64/special -python -c "import neuron; neuron.test(); neuron.test_rxd(); quit()"
       nrniv -python -c "import neuron; neuron.test(); neuron.test_rxd(); quit()"
     else
       python -c "import neuron; neuron.test(); neuron.test_rxd(); quit()"
     fi
 
-    # Test 7: run demo
+    # Test 8: run demo
     neurondemo -c 'demo(4)' -c 'run()' -c 'quit()'
 }
 
@@ -128,7 +131,7 @@ run_parallel_test() {
 test_wheel () {
     # sample mod file for nrnivmodl check
     mkdir -p tmp_mod
-    cp share/examples/nrniv/nmodl/gap.mod tmp_mod/
+    cp share/examples/nrniv/nmodl/cacum.mod tmp_mod/
 
     echo "Using `which $python_exe` : `$python_exe --version`"
     echo "=========== SERIAL TESTS ==========="

src/mswin/batch/instal.sh

@@ -250,7 +250,7 @@ unzip -d $D -o $Z
 rm $Z
 cd $B/share
 rm -f $Z
-zip -l $Z lib/nrn.defaults lib/nrnunits.lib
+zip -l $Z lib/nrn.defaults* lib/nrnunits.lib*
 unzip -d $D -o $Z 
 rm $Z
 fi

src/mswin/batch/instalmingw.sh

@@ -191,7 +191,7 @@ unzip -d $D -o $Z
 rm $Z
 cd $B/share
 rm -f $Z
-zip -l $Z lib/nrn.defaults lib/nrnunits.lib
+zip -l $Z lib/nrn.defaults* lib/nrnunits.lib*
 unzip -d $D -o $Z 
 rm $Z
 for f in $DB/hocmodule*.dll ; do