Merge pull request #1455 from SpiNNakerManchester/use_build

runs an intergation test in linux, windows and macos

.github/workflows/c_actions.yml

@@ -22,54 +22,39 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-    - name: Checkout SupportScripts
-      uses: actions/checkout@v4
+    - name: Prepare
+      uses: SpiNNakerManchester/SupportScripts/actions/prepare@main
       with:
-        repository: SpiNNakerManchester/SupportScripts
-        path: support
-    - name: Set up Python 3.12
-      # Note: Python is needed for spinn_utilities.make_tools when building
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.12
-    - name: Install Ubuntu dependencies
-      uses: ./support/actions/apt-get-install
-      with:
-        packages: doxygen gcc-arm-none-eabi
-
-    - name: Checkout SpiNNaker C Dependencies
-      uses: ./support/actions/install-spinn-deps
-      with:
-        # Note: SpiNNUtils needed for spinn_utilities.make_tools
-        repositories: spinnaker_tools spinn_common SpiNNFrontEndCommon
-        install: false
-
-    - name: "Prepare: Install SpiNNUtils"
-      uses: ./support/actions/install-spinn-deps
-      with:
-        # Note: SpiNNUtils needed for spinn_utilities.make_tools
-        repositories: SpiNNUtils
-        install: true
+        python-version: "3.12"
+        checkout_dependencies: spinnaker_tools spinn_common SpiNNFrontEndCommon
+        install_dependencies: SpiNNUtils
+        install_module: false
+        install_check_tools: false
+        ubuntu_packages: doxygen gcc-arm-none-eabi
 
     - name: Set environment variables
       run: |
         echo "SPINN_DIRS=$PWD/spinnaker_tools" >> $GITHUB_ENV
         echo "NEURAL_MODELLING_DIRS=$PWD/neural_modelling" >> $GITHUB_ENV
+        echo "C_LOGS_DICT=$PWD/logs.sqlite3" >> $GITHUB_ENV
 
     - name: Lint C code using Vera++
-      uses: ./support/actions/vera
+      uses: SpiNNakerManchester/SupportScripts/actions/vera@main
       with:
         base-dir: neural_modelling/src
 
     - name: Build C dependencies
       run: |
         make -C $SPINN_DIRS
         make -C spinn_common install
-        make -C SpiNNFrontEndCommon/c_common install
 
-    - name: Build C code
+    - name: Build FEC C code
+      run: make
+      working-directory: SpiNNFrontEndCommon/c_common
+      env:
+        CFLAGS: -fdiagnostics-color=always
+
+    - name: Build PYNN C code
       run: make
       working-directory: neural_modelling
       env:
@@ -84,3 +69,77 @@ jobs:
     - name: Build documentation using doxygen
       run: make doxysetup doxygen
       working-directory: neural_modelling
+
+    - name: Upload Fec binaries
+      uses: actions/upload-artifact@v4
+      with:
+        name: fec_binaries
+        path: SpiNNFrontEndCommon/spinn_front_end_common/common_model_binaries/*.aplx
+        retention-days: 5
+
+    - name: Upload PyNN binaries
+      uses: actions/upload-artifact@v4
+      with:
+        name: pynn_binaries
+        path: spynnaker/pyNN/model_binaries/*.aplx
+        retention-days: 5
+
+    - name: Upload log.sqlite3s
+      uses: actions/upload-artifact@v4
+      with:
+        name: logs.sqlite3
+        path: logs.sqlite3
+        retention-days: 5
+
+  test:
+    needs: build
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 20
+    strategy:
+      matrix:
+        runner: [ubuntu-latest, windows-latest, macos-latest]
+
+    steps:
+    - name: Prepare
+      uses: SpiNNakerManchester/SupportScripts/actions/prepare@main
+      with:
+        python-version: "3.12"
+        install_dependencies: > 
+          SpiNNUtils SpiNNMachine SpiNNMan PACMAN spalloc
+          SpiNNFrontEndCommon TestBase
+        install_module: true
+        install_check_tools: false
+        ubuntu_packages: graphviz
+        cfg_file: spynnaker
+
+    - name: Download Fec binaries
+      uses: actions/download-artifact@v4
+      with:
+        name: fec_binaries
+        # Need to go into spynnaker as FEC will have been moved by install
+        path: external_binaries
+
+    - name: Download PyNN Binaries
+      uses: actions/download-artifact@v4
+      with:
+        name: pynn_binaries
+        path: external_binaries
+
+    - name: Download logs.sqlite3
+      uses: actions/download-artifact@v4
+      with:
+        name: logs.sqlite3
+        path: external_binaries
+
+    - name: Append cfg
+      # Doing this in the prepare action ended with a different path
+      run: | 
+        echo '[Mapping]' >> ~/.spynnaker.cfg
+        echo "external_binaries=$PWD/external_binaries" >> ~/.spynnaker.cfg
+        cat ~/.spynnaker.cfg
+
+    - name: Test with pytest and proxy
+      env:
+        SPALLOC_USER: ${{ secrets.SPALLOC_USER }}
+        SPALLOC_PASSWORD: ${{ secrets.SPALLOC_PASSWORD }}
+      run: pytest proxy_integration_tests

.github/workflows/python_actions.yml

@@ -26,9 +26,12 @@ jobs:
   call:
     uses: SpiNNakerManchester/SupportScripts/.github/workflows/python_checks.yml@main
     with:
-      base-package: spynnaker
       dependencies: SpiNNUtils SpiNNMachine SpiNNMan PACMAN spalloc SpiNNFrontEndCommon TestBase
+      ubuntu_packages: graphviz
       test_directories: unittests
+      coverage-package: spynnaker
       flake8-packages: spynnaker unittests
+      pylint-packages: spynnaker
+      mypy-packages: spynnaker
       check_prereleases: false
       cfg_file: spynnaker

proxy_integration_tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 The University of Manchester
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

proxy_integration_tests/test_python_debug/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 The University of Manchester
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

proxy_integration_tests/test_python_debug/check_debug.py

@@ -0,0 +1,105 @@
+# Copyright (c) 2017 The University of Manchester
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from spinn_utilities.config_holder import get_config_bool
+import spinn_front_end_common.utilities.report_functions.reports as \
+    reports_names
+from spinn_front_end_common.utilities.report_functions.network_specification \
+    import _FILENAME as network_specification_file_name
+from spinn_front_end_common.utilities.report_functions.drift_report import (
+    CLOCK_DRIFT_REPORT)
+from spinn_front_end_common.utilities.report_functions.\
+    memory_map_on_host_report import _FOLDER_NAME as \
+    memory_map_on_host_report
+# from spinn_front_end_common.utilities.report_functions.energy_report \
+#    import EnergyReport
+from spinn_front_end_common.utilities.report_functions.board_chip_report \
+    import AREA_CODE_REPORT_NAME
+from spinn_front_end_common.utility_models import \
+     DataSpeedUpPacketGatherMachineVertex
+from spinnaker_testbase import BaseTestCase
+from spynnaker.pyNN.data import SpynnakerDataView
+from spynnaker.pyNN.extra_algorithms.\
+    spynnaker_neuron_network_specification_report import (
+        _GRAPH_NAME) #, _GRAPH_FORMAT)
+import pyNN.spiNNaker as sim
+
+
+class CheckDebug(BaseTestCase):
+    """
+    that it does not crash in debug mode. All reports on.
+    """
+    def debug(self):
+        # pylint: disable=protected-access
+        reports = [
+            # write_energy_report
+            # EnergyReport._DETAILED_FILENAME,
+            # EnergyReport._SUMMARY_FILENAME,
+            # write_text_specs = False
+            "data_spec_text_files",
+            # write_router_reports
+            reports_names._ROUTING_FILENAME,
+            # write_partitioner_reports
+            reports_names._PARTITIONING_FILENAME,
+            # write_application_graph_placer_report
+            reports_names._PLACEMENT_VTX_GRAPH_FILENAME,
+            reports_names._PLACEMENT_CORE_GRAPH_FILENAME,
+            reports_names._SDRAM_FILENAME,
+            # repeats reports_names._SDRAM_FILENAME,
+            # write_router_info_report
+            reports_names._VIRTKEY_FILENAME,
+            # write_routing_table_reports
+            reports_names._ROUTING_TABLE_DIR,
+            reports_names._C_ROUTING_TABLE_DIR,
+            reports_names._COMPARED_FILENAME,
+            # write_memory_map_report
+            memory_map_on_host_report,
+            # write_network_specification_report
+            network_specification_file_name,
+            "provenance_data",
+            # write_tag_allocation_reports
+            reports_names._TAGS_FILENAME,
+            # write_drift_report_end or start
+            CLOCK_DRIFT_REPORT,
+            # write_board_chip_report
+            AREA_CODE_REPORT_NAME,
+            # spynnaker_neuron_graph_network_specification_report
+            _GRAPH_NAME,
+            # _GRAPH_NAME + "." +  _GRAPH_FORMAT,
+            ]
+
+        sim.setup(1.0)
+        if (get_config_bool("Machine", "enable_advanced_monitor_support")
+                and not get_config_bool("Java", "use_java")):
+            # write_data_speed_up_report
+            reports.append(
+                DataSpeedUpPacketGatherMachineVertex.OUT_REPORT_NAME)
+            reports.append(DataSpeedUpPacketGatherMachineVertex.IN_REPORT_NAME)
+        pop = sim.Population(100, sim.IF_curr_exp, {}, label="pop")
+        pop.record("v")
+        inp = sim.Population(1, sim.SpikeSourceArray(
+            spike_times=[0]), label="input")
+        sim.Projection(inp, pop, sim.AllToAllConnector(),
+                       synapse_type=sim.StaticSynapse(weight=5))
+        sim.run(0)
+        pop.get_data("v")
+        run0 = SpynnakerDataView.get_run_dir_path()
+        found = os.listdir(run0)
+        for report in reports:
+            self.assertIn(report, found)
+        self.assertIn("data.sqlite3", found)
+        self.assertIn("ds.sqlite3", found)
+
+        sim.end()

proxy_integration_tests/test_python_debug/spynnaker.cfg

@@ -0,0 +1,10 @@
+[Machine]
+enable_advanced_monitor_support = True
+
+[Mode]
+# mode = Production or Debug
+# In Debug mode all report Boolean config values are automatically overwritten to True
+mode = Debug
+
+[Java]
+use_java = False

proxy_integration_tests/test_python_debug/test_debug.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2017 The University of Manchester
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .check_debug import CheckDebug
+
+
+class TestDebug(CheckDebug):
+
+    def test_debug(self):
+        self.runsafe(self.debug)