From 0b3bb977760c9e0eb6fbadb414a19533ee3e4e0a Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 9 Oct 2024 14:29:30 -0400 Subject: [PATCH 01/82] added catalog file for 1.1 reference simulations to /data --- ...mock_catalog_heratext_2458098.38824015.txt | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/pyuvsim/data/mock_catalog_heratext_2458098.38824015.txt diff --git a/src/pyuvsim/data/mock_catalog_heratext_2458098.38824015.txt b/src/pyuvsim/data/mock_catalog_heratext_2458098.38824015.txt new file mode 100644 index 00000000..5cc5360b --- /dev/null +++ b/src/pyuvsim/data/mock_catalog_heratext_2458098.38824015.txt @@ -0,0 +1,44 @@ +SOURCE_ID RA_ICRS [deg] Dec_ICRS [deg] Flux [Jy] Frequency [Hz] +HERATEST0 68.48535 -28.559917 1 100000000.0 +HERATEST1 66.21075 -28.669444 1 100000000.0 +HERATEST2 63.93300 -28.742866 1 100000000.0 +HERATEST3 62.79210 -28.76516 1 100000000.0 +HERATEST4 61.65180 -28.779055 1 100000000.0 +HERATEST5 59.37045 -28.778843 1 100000000.0 +HERATEST6 57.08925 -28.74223 1 100000000.0 +HERATEST7 54.81165 -28.668388 1 100000000.0 +HERATEST8 52.53720 -28.558443 1 100000000.0 +HERATEST9 68.41275 -27.564489 1 100000000.0 +HERATEST10 66.15885 -27.671835 1 100000000.0 +HERATEST11 63.90090 -27.743365 1 100000000.0 +HERATEST12 59.38125 -27.778828 1 100000000.0 +HERATEST13 58.25100 -27.765359 1 100000000.0 +HERATEST14 54.86385 -27.670802 1 100000000.0 +HERATEST15 52.60995 -27.563048 1 100000000.0 +HERATEST16 68.34300 -26.568897 1 100000000.0 +HERATEST17 67.22640 -26.625843 1 100000000.0 +HERATEST18 66.10875 -26.674063 1 100000000.0 +HERATEST19 63.87120 -26.744231 1 100000000.0 +HERATEST20 62.75160 -26.766141 1 100000000.0 +HERATEST21 59.39115 -26.779049 1 100000000.0 +HERATEST22 58.27125 -26.765736 1 100000000.0 +HERATEST23 57.15150 -26.743624 1 100000000.0 +HERATEST24 54.91395 -26.673054 1 100000000.0 +HERATEST25 53.79645 -26.624634 1 100000000.0 +HERATEST26 52.67985 -26.56749 1 100000000.0 +HERATEST27 68.27595 -25.573194 1 100000000.0 +HERATEST28 66.06075 -25.676232 1 100000000.0 +HERATEST29 63.84210 -25.745088 1 100000000.0 +HERATEST30 59.40120 -25.779269 1 100000000.0 +HERATEST31 57.18090 -25.744495 1 100000000.0 +HERATEST32 54.96225 -25.675246 1 100000000.0 +HERATEST33 52.74720 -25.571818 1 100000000.0 +HERATEST34 68.21160 -24.577407 1 100000000.0 +HERATEST35 66.01455 -24.678446 1 100000000.0 +HERATEST36 63.81510 -24.745485 1 100000000.0 +HERATEST37 62.71350 -24.76709 1 100000000.0 +HERATEST38 61.61280 -24.779435 1 100000000.0 +HERATEST39 59.41035 -24.779242 1 100000000.0 +HERATEST40 58.30965 -24.766704 1 100000000.0 +HERATEST41 57.20820 -24.744905 1 100000000.0 +HERATEST42 53.90925 -24.630704 1 100000000.0 From 6da5eb7bdb8bf16e4dcbaad6a30cead324680c05 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 9 Oct 2024 14:52:34 -0400 Subject: [PATCH 02/82] possible approach to packaging reference simulation data files --- .../obsparam_ref_1.1_uniform.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/pyuvsim/data/test_catalogs/first_generation/obsparam_ref_1.1_uniform.yaml diff --git a/src/pyuvsim/data/test_catalogs/first_generation/obsparam_ref_1.1_uniform.yaml b/src/pyuvsim/data/test_catalogs/first_generation/obsparam_ref_1.1_uniform.yaml new file mode 100644 index 00000000..70a9d730 --- /dev/null +++ b/src/pyuvsim/data/test_catalogs/first_generation/obsparam_ref_1.1_uniform.yaml @@ -0,0 +1,17 @@ +filing: + outdir: "./results_data/" + outfile_name: 'ref_1.1_uniform' + output_format: 'uvh5' +freq: + Nfreqs: 1 + channel_width: 1.0 + start_freq: 100000000.0 +sources: + catalog: "../catalog_files/mock_catalog_heratext_2458098.38824015.txt" +telescope: + array_layout: telescope_config/mwa_nocore_layout.csv + telescope_config_name: telescope_config/mwa88_nocore_config_uniform.yaml +time: + Ntimes: 1 + integration_time: 1.0 + start_time: 2458098.38824015 From 9b85f702b7bae93efe88dd48cb1f0dac3f03852d Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 9 Oct 2024 15:28:27 -0400 Subject: [PATCH 03/82] moved files to better location --- .../mock_catalog_heratext_2458098.38824015.txt | 0 .../obsparam_ref_1.1_uniform.yaml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/pyuvsim/data/{ => test_catalogs}/mock_catalog_heratext_2458098.38824015.txt (100%) rename src/pyuvsim/data/{test_catalogs/first_generation => test_config}/obsparam_ref_1.1_uniform.yaml (100%) diff --git a/src/pyuvsim/data/mock_catalog_heratext_2458098.38824015.txt b/src/pyuvsim/data/test_catalogs/mock_catalog_heratext_2458098.38824015.txt similarity index 100% rename from src/pyuvsim/data/mock_catalog_heratext_2458098.38824015.txt rename to src/pyuvsim/data/test_catalogs/mock_catalog_heratext_2458098.38824015.txt diff --git a/src/pyuvsim/data/test_catalogs/first_generation/obsparam_ref_1.1_uniform.yaml b/src/pyuvsim/data/test_config/obsparam_ref_1.1_uniform.yaml similarity index 100% rename from src/pyuvsim/data/test_catalogs/first_generation/obsparam_ref_1.1_uniform.yaml rename to src/pyuvsim/data/test_config/obsparam_ref_1.1_uniform.yaml From 5adc447b51201d5cd0ecf769416be768c412ff8f Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 9 Oct 2024 15:29:15 -0400 Subject: [PATCH 04/82] fixed up filepaths for 1.1_uniform reference simulation in /data --- .../data/test_config/obsparam_ref_1.1_uniform.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pyuvsim/data/test_config/obsparam_ref_1.1_uniform.yaml b/src/pyuvsim/data/test_config/obsparam_ref_1.1_uniform.yaml index 70a9d730..02063cd5 100644 --- a/src/pyuvsim/data/test_config/obsparam_ref_1.1_uniform.yaml +++ b/src/pyuvsim/data/test_config/obsparam_ref_1.1_uniform.yaml @@ -4,14 +4,14 @@ filing: output_format: 'uvh5' freq: Nfreqs: 1 - channel_width: 1.0 + channel_width: 80000.0 start_freq: 100000000.0 sources: - catalog: "../catalog_files/mock_catalog_heratext_2458098.38824015.txt" + catalog: "../test_catalogs/mock_catalog_heratext_2458098.38824015.txt" telescope: - array_layout: telescope_config/mwa_nocore_layout.csv - telescope_config_name: telescope_config/mwa88_nocore_config_uniform.yaml + array_layout: "../mwa_nocore_layout.csv" + telescope_config_name: "../mwa88_nocore_config.yaml" time: Ntimes: 1 - integration_time: 1.0 + integration_time: 11.0 start_time: 2458098.38824015 From 64892cf25215a5d567ece17477ae9bfd8aed2ea9 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 10 Oct 2024 09:59:51 -0400 Subject: [PATCH 05/82] attempting to add benchmarks and updated ref_sim_1.1 in /data --- .../test_config/obsparam_ref_1.1_uniform.yaml | 2 +- tests/test_run_ref.py | 67 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 tests/test_run_ref.py diff --git a/src/pyuvsim/data/test_config/obsparam_ref_1.1_uniform.yaml b/src/pyuvsim/data/test_config/obsparam_ref_1.1_uniform.yaml index 02063cd5..9048b381 100644 --- a/src/pyuvsim/data/test_config/obsparam_ref_1.1_uniform.yaml +++ b/src/pyuvsim/data/test_config/obsparam_ref_1.1_uniform.yaml @@ -1,5 +1,5 @@ filing: - outdir: "./results_data/" + outdir: "." outfile_name: 'ref_1.1_uniform' output_format: 'uvh5' freq: diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py new file mode 100644 index 00000000..a8ac8ce4 --- /dev/null +++ b/tests/test_run_ref.py @@ -0,0 +1,67 @@ +# Copyright (c) 2020 Radio Astronomy Software Group +# Licensed under the 3-clause BSD License + +import os +import warnings + +import pytest +from pyuvdata import UVData + +import pyuvsim +from pyuvsim.data import DATA_PATH as SIM_DATA_PATH + +import importlib +hasbench = importlib.util.find_spec("pytest_benchmark") is not None + +pytest.importorskip("mpi4py") # noqa + +@pytest.fixture +def goto_tempdir(tmpdir): + # Run test within temporary directory. + newpath = str(tmpdir) + cwd = os.getcwd() + os.chdir(newpath) + + yield newpath + + os.chdir(cwd) + + +#@pytest.mark.filterwarnings("ignore:antenna_diameters are not set") +#@pytest.mark.filterwarnings("ignore:Telescope Triangle is not in known_telescopes.") +#@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only") +@pytest.mark.parametrize("paramfile", ["obsparam_ref_1.1_uniform.yaml"]) +@pytest.mark.parallel(2) +@pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") +def test_run_11_uniform(benchmark, goto_tempdir, paramfile): + # Test vot and txt catalogs for parameter simulation + # Compare to reference files. + uv_ref = UVData() + uv_ref.read_uvh5(os.path.join("/oscar/data/jpober/mburdorf/pyuvsim/results_data/ref_1.1_uniform.uvh5")) + + param_filename = os.path.join(SIM_DATA_PATH, "test_config", paramfile) + # This test obsparam file has "single_source.txt" as its catalog. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + benchmark( + pyuvsim.uvsim.run_uvsim, + param_filename, + ) + #pyuvsim.uvsim.run_uvsim(param_filename) + + # Loading the file and comparing is only done on rank 0. + if pyuvsim.mpi.rank != 0: + return + + path = goto_tempdir + ofilepath = os.path.join(path, "ref_1.1_uniform.uvh5") + + uv_new = UVData.from_file(ofilepath) + + # Reset parts that will deviate + uv_new.history = uv_ref.history + #uv_ref.dut1 = uv_new.dut1 + #uv_ref.gst0 = uv_new.gst0 + #uv_ref.rdate = uv_new.rdate + + assert uv_new == uv_ref From c57da791acd84eca8e1e7c2ed3a58c2b9eac340b Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 10 Oct 2024 10:49:32 -0400 Subject: [PATCH 06/82] benchmark needs more work but almost there --- tests/test_run_ref.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index a8ac8ce4..2927edd7 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -2,7 +2,7 @@ # Licensed under the 3-clause BSD License import os -import warnings +#import warnings import pytest from pyuvdata import UVData @@ -31,23 +31,25 @@ def goto_tempdir(tmpdir): #@pytest.mark.filterwarnings("ignore:Telescope Triangle is not in known_telescopes.") #@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only") @pytest.mark.parametrize("paramfile", ["obsparam_ref_1.1_uniform.yaml"]) -@pytest.mark.parallel(2) +@pytest.mark.parallel(1) @pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") def test_run_11_uniform(benchmark, goto_tempdir, paramfile): # Test vot and txt catalogs for parameter simulation # Compare to reference files. uv_ref = UVData() - uv_ref.read_uvh5(os.path.join("/oscar/data/jpober/mburdorf/pyuvsim/results_data/ref_1.1_uniform.uvh5")) + uv_ref.read_uvh5("/oscar/data/jpober/mburdorf/pyuvsim/results_data/ref_1.1_uniform.uvh5") param_filename = os.path.join(SIM_DATA_PATH, "test_config", paramfile) # This test obsparam file has "single_source.txt" as its catalog. - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - benchmark( - pyuvsim.uvsim.run_uvsim, - param_filename, - ) - #pyuvsim.uvsim.run_uvsim(param_filename) + #with warnings.catch_warnings(): + # warnings.simplefilter("ignore") + # # not sure + benchmark( + pyuvsim.uvsim.run_uvsim, + param_filename + ) + #benchmark(print, "hi") + pyuvsim.uvsim.run_uvsim(param_filename) # Loading the file and comparing is only done on rank 0. if pyuvsim.mpi.rank != 0: @@ -55,7 +57,6 @@ def test_run_11_uniform(benchmark, goto_tempdir, paramfile): path = goto_tempdir ofilepath = os.path.join(path, "ref_1.1_uniform.uvh5") - uv_new = UVData.from_file(ofilepath) # Reset parts that will deviate @@ -65,3 +66,11 @@ def test_run_11_uniform(benchmark, goto_tempdir, paramfile): #uv_ref.rdate = uv_new.rdate assert uv_new == uv_ref + +@pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") +def test_example(benchmark): #goto_tempdir, paramfile): + #paramfile = "obsparam_ref_1.1_uniform.yaml" + # Test vot and txt catalogs for parameter simulation + # Compare to reference files. + uv_ref = UVData() + benchmark(uv_ref.read_uvh5, os.path.join("/oscar/data/jpober/mburdorf/pyuvsim/results_data/ref_1.1_uniform.uvh5")) From 19dc9af06e81661f222943cabe86ce7c570fe4a0 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 10 Oct 2024 10:55:36 -0400 Subject: [PATCH 07/82] re-commented line --- tests/test_run_ref.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 2927edd7..50b874c1 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -49,7 +49,7 @@ def test_run_11_uniform(benchmark, goto_tempdir, paramfile): param_filename ) #benchmark(print, "hi") - pyuvsim.uvsim.run_uvsim(param_filename) + #pyuvsim.uvsim.run_uvsim(param_filename) # Loading the file and comparing is only done on rank 0. if pyuvsim.mpi.rank != 0: From eec469e85ddba949e825a4f8000a7d2ab1bfe0f9 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Fri, 11 Oct 2024 08:17:49 -0400 Subject: [PATCH 08/82] preliminary commit for benchmark workflow. Still need to set up 1.1_uniform for remote downloading. --- .github/workflows/testsuite.yaml | 83 ++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index be5adfff..5fecbabb 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -208,3 +208,86 @@ jobs: file: ./coverage.xml #optional env_vars: OS,PYTHON fail_ci_if_error: true + + # TODO: should implement generic method of determining os instead of hardcoding it + benchmark: + name: Performance Benchmark + needs: tests + env: + ENV_NAME: pyuvsim_tests_mpich + PYTHON: "3.12" + runs-on: ubuntu-latest + defaults: + run: + # Adding -l {0} helps ensure conda can be found properly. + shell: bash -l {0} + steps: + - uses: actions/checkout@main + + - name: Setup Minimamba + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-variant: Mambaforge + miniforge-version: latest + use-mamba: true + python-version: ${{ env.PYTHON }} + environment-file: ci/${{ env.ENV_NAME }}.yaml + activate-environment: ${{ env.ENV_NAME }} + run-post: false + + - name: Conda Info + run: | + conda info -a + conda list + PYVER=`python -c "import sys; print('{:d}.{:d}'.format(sys.version_info.major, sys.version_info.minor))"` + if [[ $PYVER != $PYTHON ]]; then + exit 1; + fi + + # also install benchmark utility + - name: Install + run: | + pip install pytest-benchmark + pip install . + + - name: Run benchmark + run: | + mpiexec pytest --benchmark-only --benchmark-json output.json + + # Download previous benchmark result from cache (if exists) + - name: Download previous benchmark data + uses: actions/cache/restore@v4 + with: + path: ./cache + key: ubuntu-benchmark + + # Run `github-action-benchmark` action + # this step also EDITS the ./cache/benchmark-data.json file + # We do not need to add output.json to the cache directory + - name: Compare benchmarks + uses: benchmark-action/github-action-benchmark@v1 + with: + # What benchmark tool the output.txt came from + tool: 'pytest' + # Where the output from the benchmark tool is stored + output-file-path: output.json + # Where the previous data file is stored + external-data-json-path: ./cache/benchmark-data.json + # Workflow will fail when an alert happens + fail-on-alert: true + # Comment on the PR if the branch is not a fork + comment-on-alert: true + # Enable Job Summary for PRs + summary-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + + + - name: Store benchmark results + uses: actions/cache/save@v4 + # only store the cache if being run on ref sim test branch + # TODO: swap to main once functional and ready for merge + if: github.ref == 'refs/heads/1.1_ref_sim_ci_workflow' + with: + path: ./cache + key: ubuntu-benchmark + From 7d5ffd72404662f06f89e928c19eba48cd332e9f Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Sat, 12 Oct 2024 07:23:13 -0400 Subject: [PATCH 09/82] pytest should now hopefully work remotely -- file on gdrive --- tests/test_run_ref.py | 49 ++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 50b874c1..174c85de 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -4,6 +4,8 @@ import os #import warnings +import requests + import pytest from pyuvdata import UVData @@ -15,6 +17,29 @@ pytest.importorskip("mpi4py") # noqa +@pytest.fixture +def download_sims(): + target_dir = "results_data" + + if not os.path.exists(target_dir): + os.makedirs(target_dir) + + urlbase = "https://drive.google.com/uc?export=download" + + dat = [] + + fid = "1V4RmmUGrx5iH-Zyuj45p1ag3vmMuUI2A" + fname = "ref_1.1_uniform.uvh5" + # fileid name type size size_unit date time + + r = requests.get(urlbase, params={"id": fid}) + + fname = os.path.join(target_dir, fname) + + with open(fname, "wb") as ofile: + ofile.write(r.content) + + @pytest.fixture def goto_tempdir(tmpdir): # Run test within temporary directory. @@ -30,14 +55,22 @@ def goto_tempdir(tmpdir): #@pytest.mark.filterwarnings("ignore:antenna_diameters are not set") #@pytest.mark.filterwarnings("ignore:Telescope Triangle is not in known_telescopes.") #@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only") +#@pytest.mark.parallel(1) +# to test: call pytest with mpiexec / similar and see if runs @pytest.mark.parametrize("paramfile", ["obsparam_ref_1.1_uniform.yaml"]) -@pytest.mark.parallel(1) @pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") -def test_run_11_uniform(benchmark, goto_tempdir, paramfile): +def test_run_11_uniform(benchmark, goto_tempdir, download_sims, paramfile): + # download reference sim + #download_sims(goto_tempdir) + + filepath = os.path.join(goto_tempdir, "results_data", "ref_1.1_uniform.uvh5") # Test vot and txt catalogs for parameter simulation # Compare to reference files. uv_ref = UVData() - uv_ref.read_uvh5("/oscar/data/jpober/mburdorf/pyuvsim/results_data/ref_1.1_uniform.uvh5") + # TODO: implement methods to make this download from remote to appropriate path + # for now from google drive + + uv_ref.read_uvh5(filepath) param_filename = os.path.join(SIM_DATA_PATH, "test_config", paramfile) # This test obsparam file has "single_source.txt" as its catalog. @@ -48,10 +81,10 @@ def test_run_11_uniform(benchmark, goto_tempdir, paramfile): pyuvsim.uvsim.run_uvsim, param_filename ) - #benchmark(print, "hi") #pyuvsim.uvsim.run_uvsim(param_filename) # Loading the file and comparing is only done on rank 0. + # maybe comment out if pyuvsim.mpi.rank != 0: return @@ -66,11 +99,3 @@ def test_run_11_uniform(benchmark, goto_tempdir, paramfile): #uv_ref.rdate = uv_new.rdate assert uv_new == uv_ref - -@pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") -def test_example(benchmark): #goto_tempdir, paramfile): - #paramfile = "obsparam_ref_1.1_uniform.yaml" - # Test vot and txt catalogs for parameter simulation - # Compare to reference files. - uv_ref = UVData() - benchmark(uv_ref.read_uvh5, os.path.join("/oscar/data/jpober/mburdorf/pyuvsim/results_data/ref_1.1_uniform.uvh5")) From 3acf42a8ad24ad7745bcdfcfa3da5daa39bb636a Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Sat, 12 Oct 2024 08:12:36 -0400 Subject: [PATCH 10/82] trying slight change to see if all other tests run to completion properly --- tests/test_run_ref.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 174c85de..3057785b 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -4,7 +4,6 @@ import os #import warnings -import requests import pytest from pyuvdata import UVData @@ -19,6 +18,7 @@ @pytest.fixture def download_sims(): + import requests target_dir = "results_data" if not os.path.exists(target_dir): From b394a7ed283f8824c19f0e966a68bab9f56a0587 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Sat, 12 Oct 2024 08:21:16 -0400 Subject: [PATCH 11/82] updated testsuite to install requests --- .github/workflows/testsuite.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 5fecbabb..262531f4 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -244,10 +244,11 @@ jobs: exit 1; fi - # also install benchmark utility + # also install benchmark utility and requests - name: Install run: | pip install pytest-benchmark + pip install requests pip install . - name: Run benchmark From 2fe9190af37fb6813d190e6a1f9068ed9ed00703 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Tue, 15 Oct 2024 01:50:20 -0400 Subject: [PATCH 12/82] test change --- .github/workflows/testsuite.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 262531f4..79aee225 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -227,9 +227,9 @@ jobs: - name: Setup Minimamba uses: conda-incubator/setup-miniconda@v3 with: - miniforge-variant: Mambaforge + #miniforge-variant: Mambaforge miniforge-version: latest - use-mamba: true + #use-mamba: true python-version: ${{ env.PYTHON }} environment-file: ci/${{ env.ENV_NAME }}.yaml activate-environment: ${{ env.ENV_NAME }} From 046ac37209e2ce832c26e51e670b07ec291ab50c Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Tue, 15 Oct 2024 02:19:41 -0400 Subject: [PATCH 13/82] change and mpiexec --- .github/workflows/testsuite.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 79aee225..7c2716b6 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -253,7 +253,7 @@ jobs: - name: Run benchmark run: | - mpiexec pytest --benchmark-only --benchmark-json output.json + mpiexec -n 1 -np 1 pytest --benchmark-only --benchmark-json output.json # Download previous benchmark result from cache (if exists) - name: Download previous benchmark data From d995aa16b55419e43f37080cb6d52bcace993bdb Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Tue, 15 Oct 2024 02:40:39 -0400 Subject: [PATCH 14/82] tried possibly fixing caching --- .github/workflows/testsuite.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 7c2716b6..ed890123 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -260,7 +260,7 @@ jobs: uses: actions/cache/restore@v4 with: path: ./cache - key: ubuntu-benchmark + key: ubuntu-benchmark_new # Run `github-action-benchmark` action # this step also EDITS the ./cache/benchmark-data.json file @@ -290,5 +290,5 @@ jobs: if: github.ref == 'refs/heads/1.1_ref_sim_ci_workflow' with: path: ./cache - key: ubuntu-benchmark + key: ubuntu-benchmark_new From 85e1df4aef2b160a83701541becc0a1f49e6569e Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Tue, 15 Oct 2024 02:49:39 -0400 Subject: [PATCH 15/82] benchmark action stuff --- .github/workflows/testsuite.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index ed890123..57ed3bcb 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -274,12 +274,16 @@ jobs: output-file-path: output.json # Where the previous data file is stored external-data-json-path: ./cache/benchmark-data.json + # should fail consistently + alert-threshold: "101%" # Workflow will fail when an alert happens fail-on-alert: true # Comment on the PR if the branch is not a fork comment-on-alert: true # Enable Job Summary for PRs summary-always: true + # Always leave a comment + comment-always: true github-token: ${{ secrets.GITHUB_TOKEN }} From 7dae05f624a499c7d78c8be1a295ccf60fccdcd5 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 16 Oct 2024 16:14:42 -0400 Subject: [PATCH 16/82] minor updates to see if alert works and if -np 4 speeds up run --- .github/workflows/testsuite.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 57ed3bcb..19868a0f 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -253,7 +253,7 @@ jobs: - name: Run benchmark run: | - mpiexec -n 1 -np 1 pytest --benchmark-only --benchmark-json output.json + mpiexec -n 1 -np 4 pytest --benchmark-only --benchmark-json output.json # Download previous benchmark result from cache (if exists) - name: Download previous benchmark data @@ -275,7 +275,7 @@ jobs: # Where the previous data file is stored external-data-json-path: ./cache/benchmark-data.json # should fail consistently - alert-threshold: "101%" + alert-threshold: "50%" # Workflow will fail when an alert happens fail-on-alert: true # Comment on the PR if the branch is not a fork @@ -283,7 +283,7 @@ jobs: # Enable Job Summary for PRs summary-always: true # Always leave a comment - comment-always: true + # comment-always: true github-token: ${{ secrets.GITHUB_TOKEN }} From d80a216a88b7433383a3bb97b512f34c7ad5c548 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 16 Oct 2024 16:26:28 -0400 Subject: [PATCH 17/82] test 3 --- .github/workflows/testsuite.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 19868a0f..832973cf 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -212,7 +212,7 @@ jobs: # TODO: should implement generic method of determining os instead of hardcoding it benchmark: name: Performance Benchmark - needs: tests + #needs: tests env: ENV_NAME: pyuvsim_tests_mpich PYTHON: "3.12" @@ -253,7 +253,7 @@ jobs: - name: Run benchmark run: | - mpiexec -n 1 -np 4 pytest --benchmark-only --benchmark-json output.json + mpiexec -n 1 -np 3 pytest --benchmark-only --benchmark-json output.json # Download previous benchmark result from cache (if exists) - name: Download previous benchmark data From 3ef5c82f334c0e17caba02a57c89189b325d078c Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 16 Oct 2024 16:32:59 -0400 Subject: [PATCH 18/82] just alert --- .github/workflows/testsuite.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 832973cf..4d76fcfa 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -253,7 +253,7 @@ jobs: - name: Run benchmark run: | - mpiexec -n 1 -np 3 pytest --benchmark-only --benchmark-json output.json + mpiexec -n 1 -np 1 pytest --benchmark-only --benchmark-json output.json # Download previous benchmark result from cache (if exists) - name: Download previous benchmark data From d38f00e296cdd309bdaa64c01411cc042c225f05 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 16 Oct 2024 16:48:06 -0400 Subject: [PATCH 19/82] test failed assert --- .github/workflows/testsuite.yaml | 2 -- tests/test_run_ref.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 4d76fcfa..ac753feb 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -286,7 +286,6 @@ jobs: # comment-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Store benchmark results uses: actions/cache/save@v4 # only store the cache if being run on ref sim test branch @@ -295,4 +294,3 @@ jobs: with: path: ./cache key: ubuntu-benchmark_new - diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 3057785b..195797e3 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -98,4 +98,4 @@ def test_run_11_uniform(benchmark, goto_tempdir, download_sims, paramfile): #uv_ref.gst0 = uv_new.gst0 #uv_ref.rdate = uv_new.rdate - assert uv_new == uv_ref + assert uv_new != uv_ref From 18b1145aa885fec8d8da601d89844dcd75e09787 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 16 Oct 2024 16:54:00 -0400 Subject: [PATCH 20/82] swap back to == --- tests/test_run_ref.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 195797e3..3057785b 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -98,4 +98,4 @@ def test_run_11_uniform(benchmark, goto_tempdir, download_sims, paramfile): #uv_ref.gst0 = uv_new.gst0 #uv_ref.rdate = uv_new.rdate - assert uv_new != uv_ref + assert uv_new == uv_ref From 78638e0583c38f87104fd91c2656a2a10cd67ca2 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 16 Oct 2024 17:06:53 -0400 Subject: [PATCH 21/82] swapping back --- .github/workflows/testsuite.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index ac753feb..cd93efc0 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -212,7 +212,7 @@ jobs: # TODO: should implement generic method of determining os instead of hardcoding it benchmark: name: Performance Benchmark - #needs: tests + needs: tests env: ENV_NAME: pyuvsim_tests_mpich PYTHON: "3.12" @@ -275,7 +275,7 @@ jobs: # Where the previous data file is stored external-data-json-path: ./cache/benchmark-data.json # should fail consistently - alert-threshold: "50%" + alert-threshold: "120%" # Workflow will fail when an alert happens fail-on-alert: true # Comment on the PR if the branch is not a fork @@ -283,7 +283,7 @@ jobs: # Enable Job Summary for PRs summary-always: true # Always leave a comment - # comment-always: true + comment-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - name: Store benchmark results From 41f37594a6ac79592fd5961d407f0d0292ce4f65 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 23 Oct 2024 13:46:42 -0400 Subject: [PATCH 22/82] small TODO --- tests/test_run_ref.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 3057785b..486a90c8 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -16,6 +16,11 @@ pytest.importorskip("mpi4py") # noqa +#paramfile (fix name) = ["a", "b",] +# FIXME: example for that in lunar stuff (pyuvdata in pyuvdata/test_utils/test_coordinates.py (maybe inaccurate locationing)) + +# conftest implementation -- setup and teardown happens here +# fixture can run previous fixture @pytest.fixture def download_sims(): import requests @@ -98,4 +103,6 @@ def test_run_11_uniform(benchmark, goto_tempdir, download_sims, paramfile): #uv_ref.gst0 = uv_new.gst0 #uv_ref.rdate = uv_new.rdate + # TODO: implement better asserts + assert uv_new == uv_ref From 441589f0017a781c7260396dbfde7bd9cce9c523 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 23 Oct 2024 17:59:50 -0400 Subject: [PATCH 23/82] started adding new ref sim tests --- tests/test_run_ref.py | 78 +++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 486a90c8..443afbbc 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -16,11 +16,23 @@ pytest.importorskip("mpi4py") # noqa -#paramfile (fix name) = ["a", "b",] -# FIXME: example for that in lunar stuff (pyuvdata in pyuvdata/test_utils/test_coordinates.py (maybe inaccurate locationing)) - -# conftest implementation -- setup and teardown happens here -# fixture can run previous fixture +# TODO: get running for current 8 sims (add necessary data files) +# AND DO THE FIX FOR THE GOOGLE DRIVE (UPLOAD THE FILES AND +# GET THEM TO DOWNLOAD (I GUESS TEMP JUST MAKE ANOTHER +# DICT OR FILE OR DATA STRUCTURE THAT CAN GRAB THE FILES) + +# TODO: check syntax preference for global lists! +ci_ref_sims = [ + "1.1_uniform", + "1.1_hera", + "1.1_gauss", + "1.2_uniform", + "1.2_hera", + "1.3_uniform", + "1.3_gauss" + ] + +# TODO: swap to something more permanent! @pytest.fixture def download_sims(): import requests @@ -57,52 +69,54 @@ def goto_tempdir(tmpdir): os.chdir(cwd) -#@pytest.mark.filterwarnings("ignore:antenna_diameters are not set") -#@pytest.mark.filterwarnings("ignore:Telescope Triangle is not in known_telescopes.") -#@pytest.mark.filterwarnings("ignore:Fixing auto-correlations to be be real-only") -#@pytest.mark.parallel(1) -# to test: call pytest with mpiexec / similar and see if runs -@pytest.mark.parametrize("paramfile", ["obsparam_ref_1.1_uniform.yaml"]) +# TODO: add all 8 yaml files +@pytest.mark.parametrize("paramfile", ci_ref_sims) @pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") def test_run_11_uniform(benchmark, goto_tempdir, download_sims, paramfile): # download reference sim #download_sims(goto_tempdir) - filepath = os.path.join(goto_tempdir, "results_data", "ref_1.1_uniform.uvh5") - # Test vot and txt catalogs for parameter simulation - # Compare to reference files. - uv_ref = UVData() - # TODO: implement methods to make this download from remote to appropriate path - # for now from google drive + # construct filename and filepath of downloaded file + uvh5_filename = "ref_" + paramfile + ".uvh5" + uvh5_filepath = os.path.join(goto_tempdir, "results_data", uvh5_filename) + # instantiate UVData object and read in the downloaded uvh5 + # file as the point of historical comparison + uv_ref = UVData() uv_ref.read_uvh5(filepath) - param_filename = os.path.join(SIM_DATA_PATH, "test_config", paramfile) - # This test obsparam file has "single_source.txt" as its catalog. - #with warnings.catch_warnings(): - # warnings.simplefilter("ignore") - # # not sure + # construct filename and filepath of yaml file used + # to run simulation + yaml_filename = "obsparam_ref_" + paramfile + ".yaml" + yaml_filepath = os.path.join(SIM_DATA_PATH, "test_config", yaml_filename) + + # benchmark uvsim.run_uvsim method with param_filename argument + # runs around 10 times to get benchmark data + # outdir is given by the yaml file but should be current working directory + # for all the reference simulations + # TODO: think more about filepath specification, also there was a warning I can get rid of probably + # TODO: check if somehow making like 10 copies of output accidentally (whoops) benchmark( pyuvsim.uvsim.run_uvsim, - param_filename + yaml_filepath ) - #pyuvsim.uvsim.run_uvsim(param_filename) - # Loading the file and comparing is only done on rank 0. - # maybe comment out + # loading the file and comparing is only done on rank 0. if pyuvsim.mpi.rank != 0: return + # instantiate new UVData object from the benchmark run_uvsim output + # as current point of comparison path = goto_tempdir - ofilepath = os.path.join(path, "ref_1.1_uniform.uvh5") + ofilepath = os.path.join(path, uvh5_filename) uv_new = UVData.from_file(ofilepath) - # Reset parts that will deviate + # reset part(s) that should deviate + # TODO: maybe assert that deviation occurred uv_new.history = uv_ref.history - #uv_ref.dut1 = uv_new.dut1 - #uv_ref.gst0 = uv_new.gst0 - #uv_ref.rdate = uv_new.rdate + # perform equality check between historical and current reference + # simulation output # TODO: implement better asserts - + # include lower tolerance for deviations assert uv_new == uv_ref From b5bb17bab16e2d6bbdcc2139b46db394ad86928a Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 23 Oct 2024 21:47:56 -0400 Subject: [PATCH 24/82] formatting --- .github/workflows/testsuite.yaml | 4 +- tests/test_run_ref.py | 65 +++++++++++++++----------------- 2 files changed, 33 insertions(+), 36 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index cd93efc0..2ef4657e 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -208,7 +208,7 @@ jobs: file: ./coverage.xml #optional env_vars: OS,PYTHON fail_ci_if_error: true - + # TODO: should implement generic method of determining os instead of hardcoding it benchmark: name: Performance Benchmark @@ -288,7 +288,7 @@ jobs: - name: Store benchmark results uses: actions/cache/save@v4 - # only store the cache if being run on ref sim test branch + # only store the cache if being run on ref sim test branch # TODO: swap to main once functional and ready for merge if: github.ref == 'refs/heads/1.1_ref_sim_ci_workflow' with: diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 443afbbc..4c859a7e 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -1,50 +1,52 @@ # Copyright (c) 2020 Radio Astronomy Software Group # Licensed under the 3-clause BSD License +import importlib import os -#import warnings - import pytest from pyuvdata import UVData -import pyuvsim from pyuvsim.data import DATA_PATH as SIM_DATA_PATH +from pyuvsim.mpi import rank +from pyuvsim.uvsim import run_uvsim -import importlib hasbench = importlib.util.find_spec("pytest_benchmark") is not None pytest.importorskip("mpi4py") # noqa # TODO: get running for current 8 sims (add necessary data files) -# AND DO THE FIX FOR THE GOOGLE DRIVE (UPLOAD THE FILES AND -# GET THEM TO DOWNLOAD (I GUESS TEMP JUST MAKE ANOTHER +# AND DO THE FIX FOR THE GOOGLE DRIVE (UPLOAD THE FILES AND +# GET THEM TO DOWNLOAD (I GUESS TEMP JUST MAKE ANOTHER # DICT OR FILE OR DATA STRUCTURE THAT CAN GRAB THE FILES) # TODO: check syntax preference for global lists! ci_ref_sims = [ - "1.1_uniform", - "1.1_hera", - "1.1_gauss", - "1.2_uniform", - "1.2_hera", - "1.3_uniform", - "1.3_gauss" - ] + "1.1_uniform", + "1.1_hera", + "1.1_gauss", + "1.2_uniform", + "1.2_hera", + "1.3_uniform", + "1.3_gauss", +] + +# TODO: hardset it using dictionary or something # TODO: swap to something more permanent! @pytest.fixture def download_sims(): import requests - target_dir = "results_data" + + target_dir = "results_data" if not os.path.exists(target_dir): os.makedirs(target_dir) urlbase = "https://drive.google.com/uc?export=download" - dat = [] - + # TODO: upload all other files and hardcode + # them via a dictionary with ci_ref_sims for keys fid = "1V4RmmUGrx5iH-Zyuj45p1ag3vmMuUI2A" fname = "ref_1.1_uniform.uvh5" # fileid name type size size_unit date time @@ -70,22 +72,21 @@ def goto_tempdir(tmpdir): # TODO: add all 8 yaml files -@pytest.mark.parametrize("paramfile", ci_ref_sims) +@pytest.mark.parametrize("paramfile", ci_ref_sims) @pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") def test_run_11_uniform(benchmark, goto_tempdir, download_sims, paramfile): # download reference sim - #download_sims(goto_tempdir) + # download_sims(goto_tempdir) # construct filename and filepath of downloaded file uvh5_filename = "ref_" + paramfile + ".uvh5" uvh5_filepath = os.path.join(goto_tempdir, "results_data", uvh5_filename) - # instantiate UVData object and read in the downloaded uvh5 + # instantiate UVData object and read in the downloaded uvh5 # file as the point of historical comparison - uv_ref = UVData() - uv_ref.read_uvh5(filepath) + uv_ref = UVData.from_file(uvh5_filepath) - # construct filename and filepath of yaml file used + # construct filename and filepath of yaml file used # to run simulation yaml_filename = "obsparam_ref_" + paramfile + ".yaml" yaml_filepath = os.path.join(SIM_DATA_PATH, "test_config", yaml_filename) @@ -94,25 +95,21 @@ def test_run_11_uniform(benchmark, goto_tempdir, download_sims, paramfile): # runs around 10 times to get benchmark data # outdir is given by the yaml file but should be current working directory # for all the reference simulations - # TODO: think more about filepath specification, also there was a warning I can get rid of probably + # TODO: think more about filepath specification, also there was a warning # TODO: check if somehow making like 10 copies of output accidentally (whoops) - benchmark( - pyuvsim.uvsim.run_uvsim, - yaml_filepath - ) + benchmark(run_uvsim, yaml_filepath) # loading the file and comparing is only done on rank 0. - if pyuvsim.mpi.rank != 0: + if rank != 0: return # instantiate new UVData object from the benchmark run_uvsim output # as current point of comparison - path = goto_tempdir - ofilepath = os.path.join(path, uvh5_filename) - uv_new = UVData.from_file(ofilepath) + new_uvh5_filepath = os.path.join(goto_tempdir, uvh5_filename) + uv_new = UVData.from_file(new_uvh5_filepath) - # reset part(s) that should deviate - # TODO: maybe assert that deviation occurred + # fix part(s) that should deviate + # TODO: maybe assert that deviation occurred uv_new.history = uv_ref.history # perform equality check between historical and current reference From 98e01447496cca7cd20dcbea979125c8fdea83ba Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 30 Oct 2024 05:56:17 -0400 Subject: [PATCH 25/82] added 1.1_gauss --- .../data/mwa88_nocore_config_gauss.yaml | 5 ++ .../test_config/obsparam_ref_1.1_gauss.yaml | 18 +++++ tests/test_run_ref.py | 68 +++++++++++++++---- 3 files changed, 79 insertions(+), 12 deletions(-) create mode 100644 src/pyuvsim/data/mwa88_nocore_config_gauss.yaml create mode 100644 src/pyuvsim/data/test_config/obsparam_ref_1.1_gauss.yaml diff --git a/src/pyuvsim/data/mwa88_nocore_config_gauss.yaml b/src/pyuvsim/data/mwa88_nocore_config_gauss.yaml new file mode 100644 index 00000000..21e81e29 --- /dev/null +++ b/src/pyuvsim/data/mwa88_nocore_config_gauss.yaml @@ -0,0 +1,5 @@ +beam_paths: + 0 : 'gaussian' +sigma: 0.08449 +telescope_location: (-30.72152777777791, 21.428305555555557, 1073.0000000093132) +telescope_name: MWA diff --git a/src/pyuvsim/data/test_config/obsparam_ref_1.1_gauss.yaml b/src/pyuvsim/data/test_config/obsparam_ref_1.1_gauss.yaml new file mode 100644 index 00000000..1c8e8f1d --- /dev/null +++ b/src/pyuvsim/data/test_config/obsparam_ref_1.1_gauss.yaml @@ -0,0 +1,18 @@ +filing: + outdir: "." + outfile_name: 'ref_1.1_gauss' + output_format: 'uvh5' +freq: + Nfreqs: 1 + channel_width: 80000.0 + start_freq: 100000000.0 +sources: + catalog: "../test_catalogs/mock_catalog_heratext_2458098.38824015.txt" +telescope: + array_layout: "../mwa_nocore_layout.csv" + telescope_config_name: "../mwa88_nocore_config_gauss.yaml" +time: +# Ntimes: 10 + Ntimes: 1 + integration_time: 11.0 + start_time: 2458098.38824015 diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 4c859a7e..415ce49e 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -23,12 +23,13 @@ # TODO: check syntax preference for global lists! ci_ref_sims = [ "1.1_uniform", - "1.1_hera", "1.1_gauss", - "1.2_uniform", - "1.2_hera", - "1.3_uniform", - "1.3_gauss", + #"1.1_hera", + #"1.1_gauss", + #"1.2_uniform", + #"1.2_hera", + #"1.3_uniform", + #"1.3_gauss", ] @@ -43,20 +44,59 @@ def download_sims(): if not os.path.exists(target_dir): os.makedirs(target_dir) + # temporary way to store all the fileids + # control which simulations via global ci_ref_sims array + fids = { + "1.1_uniform":"1V4RmmUGrx5iH-Zyuj45p1ag3vmMuUI2A", + "1.1_gauss":"1gTj9BSunEoyde5Dq8a5IhUieMo9PuMuC", + #"1.1_hera":"13B0YOSO9v4KsEGX9GVHuBEQrt5Tnjsxr", + #"1.2_gauss":a, + #"1.2_uniform":a, + #"1.2_hera":a, + #"1.3_uniform":a, + #"1.3_gauss":a + } + + urlbase = "https://drive.google.com/uc?export=download" + + # for each sim name in ci_ref_sims, checks that needs hera uvbeam and + # has downloadable data (is a key to fids) + download_hera_uvbeam = any(["hera" in sim for sim in ci_ref_sims if sim in fids]) + print(download_hera_uvbeam) + + # download the hera uvbeam data file if we need it + if download_hera_uvbeam: + print("skipping as file cannot currently be downloaded") + + #fid = "1lqkLlnB3uE17FcPB2GcJJQoCcn7-wVP8" + #fname = os.path.join(target_dir, "HERA_NicCST_fullfreq.uvbeam") + #r = requests.get(urlbase, params={"id": fid}) + # write the file + #with open(fname, "wb") as ofile: + # ofile.write(r.content) # TODO: upload all other files and hardcode # them via a dictionary with ci_ref_sims for keys - fid = "1V4RmmUGrx5iH-Zyuj45p1ag3vmMuUI2A" - fname = "ref_1.1_uniform.uvh5" - # fileid name type size size_unit date time + for sim in ci_ref_sims: + if sim not in fids.keys(): + print(f"key {sim} not found!") + continue - r = requests.get(urlbase, params={"id": fid}) + # get id from dict + fid = fids[sim] + + # get content + r = requests.get(urlbase, params={"id": fid}) - fname = os.path.join(target_dir, fname) + # set filename with full filepath + fname = "ref_" + sim + ".uvh5" + fname = os.path.join(target_dir, fname) - with open(fname, "wb") as ofile: - ofile.write(r.content) + # write the file + with open(fname, "wb") as ofile: + print(f"writing {fname}") + ofile.write(r.content) @pytest.fixture @@ -99,6 +139,10 @@ def test_run_11_uniform(benchmark, goto_tempdir, download_sims, paramfile): # TODO: check if somehow making like 10 copies of output accidentally (whoops) benchmark(run_uvsim, yaml_filepath) + print(f"\nfor run with {paramfile}:\n") + print(f"\n\ncurrent working directory is: {os.getcwd()}\n\n") + print(f"\n\nfiles: {os.listdir()}\n\n") + # loading the file and comparing is only done on rank 0. if rank != 0: return From 8d13bf2968a63b0f13d3e214c46790b2fc39e397 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 30 Oct 2024 08:04:04 -0400 Subject: [PATCH 26/82] got 1.1 uniform and gauss working, and resolved warnings --- src/pyuvsim/data/mwa88_nocore_config_gauss.yaml | 5 +++-- .../data/test_config/obsparam_ref_1.1_gauss.yaml | 12 +++++++----- .../data/test_config/obsparam_ref_1.1_uniform.yaml | 11 +++++++---- tests/test_run_ref.py | 13 ++++++------- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/pyuvsim/data/mwa88_nocore_config_gauss.yaml b/src/pyuvsim/data/mwa88_nocore_config_gauss.yaml index 21e81e29..b3aab3d3 100644 --- a/src/pyuvsim/data/mwa88_nocore_config_gauss.yaml +++ b/src/pyuvsim/data/mwa88_nocore_config_gauss.yaml @@ -1,5 +1,6 @@ beam_paths: - 0 : 'gaussian' -sigma: 0.08449 + 0: + type: gaussian + sigma: 0.08449 telescope_location: (-30.72152777777791, 21.428305555555557, 1073.0000000093132) telescope_name: MWA diff --git a/src/pyuvsim/data/test_config/obsparam_ref_1.1_gauss.yaml b/src/pyuvsim/data/test_config/obsparam_ref_1.1_gauss.yaml index 1c8e8f1d..bba05d8f 100644 --- a/src/pyuvsim/data/test_config/obsparam_ref_1.1_gauss.yaml +++ b/src/pyuvsim/data/test_config/obsparam_ref_1.1_gauss.yaml @@ -1,5 +1,5 @@ filing: - outdir: "." + outdir: '.' outfile_name: 'ref_1.1_gauss' output_format: 'uvh5' freq: @@ -7,12 +7,14 @@ freq: channel_width: 80000.0 start_freq: 100000000.0 sources: - catalog: "../test_catalogs/mock_catalog_heratext_2458098.38824015.txt" + catalog: '../test_catalogs/mock_catalog_heratext_2458098.38824015.txt' telescope: - array_layout: "../mwa_nocore_layout.csv" - telescope_config_name: "../mwa88_nocore_config_gauss.yaml" + array_layout: '../mwa_nocore_layout.csv' + telescope_config_name: '../mwa88_nocore_config_gauss.yaml' time: -# Ntimes: 10 Ntimes: 1 integration_time: 11.0 start_time: 2458098.38824015 +ordering: + conjugation_convention: ant1 Date: Wed, 30 Oct 2024 08:23:20 -0400 Subject: [PATCH 27/82] cosmetic update to testsuite --- .github/workflows/testsuite.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 2ef4657e..43b33fd6 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -224,12 +224,10 @@ jobs: steps: - uses: actions/checkout@main - - name: Setup Minimamba + - name: Setup Miniforge uses: conda-incubator/setup-miniconda@v3 with: - #miniforge-variant: Mambaforge miniforge-version: latest - #use-mamba: true python-version: ${{ env.PYTHON }} environment-file: ci/${{ env.ENV_NAME }}.yaml activate-environment: ${{ env.ENV_NAME }} From adb9a3443f682c5a1b15f9bd51753d14336f8050 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 30 Oct 2024 08:30:30 -0400 Subject: [PATCH 28/82] slight changes to test_run_ref.py --- tests/test_run_ref.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 3c802c75..ad741361 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -21,7 +21,9 @@ # GET THEM TO DOWNLOAD (I GUESS TEMP JUST MAKE ANOTHER # DICT OR FILE OR DATA STRUCTURE THAT CAN GRAB THE FILES) -# TODO: check syntax preference for global lists! +# global list of reference simulation names +# to be used with the dictionary of gdrive file ids in download_sims, +# and to construct yaml and output filenames ci_ref_sims = [ "1.1_uniform", "1.1_gauss", @@ -33,9 +35,8 @@ #"1.3_gauss", ] - -# TODO: hardset it using dictionary or something # TODO: swap to something more permanent! +# TODO: double confirm downloading is only happening once @pytest.fixture def download_sims(): import requests @@ -58,17 +59,16 @@ def download_sims(): #"1.3_gauss":a } - urlbase = "https://drive.google.com/uc?export=download" # for each sim name in ci_ref_sims, checks that needs hera uvbeam and # has downloadable data (is a key to fids) download_hera_uvbeam = any(["hera" in sim for sim in ci_ref_sims if sim in fids]) - print(download_hera_uvbeam) # download the hera uvbeam data file if we need it if download_hera_uvbeam: - print("skipping as file cannot currently be downloaded") + # skipping as file cannot currently be downloaded + pass #fid = "1lqkLlnB3uE17FcPB2GcJJQoCcn7-wVP8" #fname = os.path.join(target_dir, "HERA_NicCST_fullfreq.uvbeam") @@ -81,7 +81,6 @@ def download_sims(): # them via a dictionary with ci_ref_sims for keys for sim in ci_ref_sims: if sim not in fids.keys(): - print(f"key {sim} not found!") continue # get id from dict @@ -96,7 +95,6 @@ def download_sims(): # write the file with open(fname, "wb") as ofile: - print(f"writing {fname}") ofile.write(r.content) @@ -137,8 +135,8 @@ def test_run_11(benchmark, goto_tempdir, download_sims, paramfile): # runs around 10 times to get benchmark data # outdir is given by the yaml file but should be current working directory # for all the reference simulations - # TODO: think more about filepath specification, also there was a warning - # TODO: check if somehow making like 10 copies of output accidentally (whoops) + # TODO: currently the benchmark ends up writing like 10 copies of the same file + # should probably fix benchmark(run_uvsim, yaml_filepath) # loading the file and comparing is only done on rank 0. From 17bd942c06da8db177550ef60646fb1c7b21277e Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 30 Oct 2024 09:08:41 -0400 Subject: [PATCH 29/82] swapped to no longer writing the UVData objects and just returning one --- tests/test_run_ref.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index ad741361..d217e9c6 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -132,23 +132,15 @@ def test_run_11(benchmark, goto_tempdir, download_sims, paramfile): assert os.path.exists(yaml_filepath) # benchmark uvsim.run_uvsim method with param_filename argument - # runs around 10 times to get benchmark data - # outdir is given by the yaml file but should be current working directory - # for all the reference simulations - # TODO: currently the benchmark ends up writing like 10 copies of the same file - # should probably fix - benchmark(run_uvsim, yaml_filepath) + # runs multiple times to get benchmark data + # outdir is given by the yaml file but should not write anything + # and simply return a UVData object + uv_new = benchmark(run_uvsim, yaml_filepath, return_uv=True) # loading the file and comparing is only done on rank 0. if pyuvsim.mpi.rank != 0: return - # instantiate new UVData object from the benchmark run_uvsim output - # as current point of comparison - new_uvh5_filepath = os.path.join(goto_tempdir, uvh5_filename) - assert os.path.exists(new_uvh5_filepath) - uv_new = UVData.from_file(new_uvh5_filepath) - # fix part(s) that should deviate # TODO: maybe assert that deviation occurred uv_new.history = uv_ref.history From f3dc61f1b284cdd2c16d7466b47153adf7c44eac Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 31 Oct 2024 16:22:39 -0400 Subject: [PATCH 30/82] changed gaussian beam to have proper syntax as well --- src/pyuvsim/data/mwa88_nocore_config_gauss.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pyuvsim/data/mwa88_nocore_config_gauss.yaml b/src/pyuvsim/data/mwa88_nocore_config_gauss.yaml index b3aab3d3..892b71fb 100644 --- a/src/pyuvsim/data/mwa88_nocore_config_gauss.yaml +++ b/src/pyuvsim/data/mwa88_nocore_config_gauss.yaml @@ -1,6 +1,6 @@ beam_paths: - 0: - type: gaussian - sigma: 0.08449 + 0 : !AnalyticBeam + class: GaussianBeam + sigma: 0.08449 telescope_location: (-30.72152777777791, 21.428305555555557, 1073.0000000093132) telescope_name: MWA From 54281910d2886573411b22b922e97d00eb5fff93 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 31 Oct 2024 19:19:33 -0400 Subject: [PATCH 31/82] preliminary attempt at sequential execution using matrix-lock --- .github/workflows/testsuite.yaml | 48 ++++++++++++++++++++++++++++++-- tests/test_run_ref.py | 10 +++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 43b33fd6..8956dd13 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -217,12 +217,44 @@ jobs: ENV_NAME: pyuvsim_tests_mpich PYTHON: "3.12" runs-on: ubuntu-latest + + strategy: + matrix: + include: + - id: test_run_11 + name: "1.1 Reference Simulations" + - id: test_run_12 + name: "1.2 Reference Simulations" + - id: test_run_13 + name: "1.3 Reference Simulations" + defaults: run: # Adding -l {0} helps ensure conda can be found properly. shell: bash -l {0} steps: - uses: actions/checkout@main + + ##################### + # Matrix lock start # + ##################### + + - name: "Initialize matrix lock" + if: matrix.id == test_run_11 + uses: rakles/matrix-lock@v1 + with: + step: init + order: "test_run_11,test_run_13,test_run_12" + + - name: "Wait for matrix lock" + uses: rakles/matrix-lock@v1 + with: + step: wait + id: ${{ matrix.id }} + + ############# + # Job Steps # + ############# - name: Setup Miniforge uses: conda-incubator/setup-miniconda@v3 @@ -251,14 +283,14 @@ jobs: - name: Run benchmark run: | - mpiexec -n 1 -np 1 pytest --benchmark-only --benchmark-json output.json + mpiexec -n 1 -np 1 pytest -k ${{ matrix.id }} --benchmark-only --benchmark-json output.json # Download previous benchmark result from cache (if exists) - name: Download previous benchmark data uses: actions/cache/restore@v4 with: path: ./cache - key: ubuntu-benchmark_new + key: ubuntu-benchmark_new_2 # Run `github-action-benchmark` action # this step also EDITS the ./cache/benchmark-data.json file @@ -291,4 +323,14 @@ jobs: if: github.ref == 'refs/heads/1.1_ref_sim_ci_workflow' with: path: ./cache - key: ubuntu-benchmark_new + key: ubuntu-benchmark_new_2 + + ######################## + # Matrix Lock Continue # + ######################## + + - name: "Continue matrix lock" + uses: rakles/matrix-lock@v1 + with: + step: continue + id: ${{ matrix.id }} diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index d217e9c6..707f7153 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -150,3 +150,13 @@ def test_run_11(benchmark, goto_tempdir, download_sims, paramfile): # TODO: implement better asserts # include lower tolerance for deviations assert uv_new == uv_ref + +@pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") +def test_run_12(benchmark, goto_tempdir): + output = benchmark(sum, [1,1]) + assert output == 2 + +@pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") +def test_run_13(benchmark, goto_tempdir): + output = benchmark(sum, [1,1]) + assert output == 2 From 8a5111f83524ea021b5d43a989888e1391d79411 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 31 Oct 2024 19:32:57 -0400 Subject: [PATCH 32/82] hopefully fixes the issue --- .github/workflows/testsuite.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 8956dd13..8eb3b19f 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -240,7 +240,7 @@ jobs: ##################### - name: "Initialize matrix lock" - if: matrix.id == test_run_11 + if: matrix.id == "test_run_11" uses: rakles/matrix-lock@v1 with: step: init From 5afe2dfaac809013af6ab96983ca12e355419240 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 31 Oct 2024 19:34:22 -0400 Subject: [PATCH 33/82] 2nd attempt --- .github/workflows/testsuite.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 8eb3b19f..61ac402d 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -240,7 +240,7 @@ jobs: ##################### - name: "Initialize matrix lock" - if: matrix.id == "test_run_11" + if: matrix.id == 'test_run_11' uses: rakles/matrix-lock@v1 with: step: init From a6579d4171e335b2e32c771f1ef0859258338df4 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 31 Oct 2024 19:45:08 -0400 Subject: [PATCH 34/82] had bad version of matrix-lock --- .github/workflows/testsuite.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 61ac402d..eead4768 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -241,13 +241,13 @@ jobs: - name: "Initialize matrix lock" if: matrix.id == 'test_run_11' - uses: rakles/matrix-lock@v1 + uses: rakles/matrix-lock@v1.0.0 with: step: init order: "test_run_11,test_run_13,test_run_12" - name: "Wait for matrix lock" - uses: rakles/matrix-lock@v1 + uses: rakles/matrix-lock@v1.0.0 with: step: wait id: ${{ matrix.id }} @@ -330,7 +330,7 @@ jobs: ######################## - name: "Continue matrix lock" - uses: rakles/matrix-lock@v1 + uses: rakles/matrix-lock@v1.0.0 with: step: continue id: ${{ matrix.id }} From 637330553c67f45a19cb4cf61a74bbd7376face3 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 31 Oct 2024 20:33:32 -0400 Subject: [PATCH 35/82] removed matrix-lock --- .github/workflows/testsuite.yaml | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index eead4768..2cf4b54b 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -219,6 +219,7 @@ jobs: runs-on: ubuntu-latest strategy: + max-parallel: 1 matrix: include: - id: test_run_11 @@ -235,27 +236,6 @@ jobs: steps: - uses: actions/checkout@main - ##################### - # Matrix lock start # - ##################### - - - name: "Initialize matrix lock" - if: matrix.id == 'test_run_11' - uses: rakles/matrix-lock@v1.0.0 - with: - step: init - order: "test_run_11,test_run_13,test_run_12" - - - name: "Wait for matrix lock" - uses: rakles/matrix-lock@v1.0.0 - with: - step: wait - id: ${{ matrix.id }} - - ############# - # Job Steps # - ############# - - name: Setup Miniforge uses: conda-incubator/setup-miniconda@v3 with: @@ -324,13 +304,3 @@ jobs: with: path: ./cache key: ubuntu-benchmark_new_2 - - ######################## - # Matrix Lock Continue # - ######################## - - - name: "Continue matrix lock" - uses: rakles/matrix-lock@v1.0.0 - with: - step: continue - id: ${{ matrix.id }} From 7b65e91ba7ee9814f179df8dc463485b049d0e7f Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Fri, 1 Nov 2024 02:47:42 -0400 Subject: [PATCH 36/82] attempt to fix up caching --- .github/workflows/testsuite.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 2cf4b54b..37f49d9c 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -265,14 +265,23 @@ jobs: run: | mpiexec -n 1 -np 1 pytest -k ${{ matrix.id }} --benchmark-only --benchmark-json output.json + # TODO: figure out why / if okay to delete before downloading? + - name: Clear previous benchmark report + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/1.1_ref_sim_ci_workflow'}} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh cache delete ${{ matrix.id }}-benchmark --repo RadioAstronomySoftwareGroup/pyuvsim + continue-on-error: true + # Download previous benchmark result from cache (if exists) - name: Download previous benchmark data - uses: actions/cache/restore@v4 + uses: actions/cache@v4 with: path: ./cache - key: ubuntu-benchmark_new_2 + key: ${{ matrix.id }}-benchmark - # Run `github-action-benchmark` action + # TODO: figure out if this appropriately updates the cache # this step also EDITS the ./cache/benchmark-data.json file # We do not need to add output.json to the cache directory - name: Compare benchmarks @@ -295,12 +304,3 @@ jobs: # Always leave a comment comment-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Store benchmark results - uses: actions/cache/save@v4 - # only store the cache if being run on ref sim test branch - # TODO: swap to main once functional and ready for merge - if: github.ref == 'refs/heads/1.1_ref_sim_ci_workflow' - with: - path: ./cache - key: ubuntu-benchmark_new_2 From 00b37157ad8aea2296b3f739ceff4e81ab1964f2 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Fri, 1 Nov 2024 03:03:16 -0400 Subject: [PATCH 37/82] dummy commit --- .github/workflows/testsuite.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 37f49d9c..a4ff680a 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -282,6 +282,7 @@ jobs: key: ${{ matrix.id }}-benchmark # TODO: figure out if this appropriately updates the cache + # 1 # this step also EDITS the ./cache/benchmark-data.json file # We do not need to add output.json to the cache directory - name: Compare benchmarks From d9edfb0a93b2203b8bf7c8f523c050327443df4a Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 7 Nov 2024 07:15:09 -0500 Subject: [PATCH 38/82] swapped back to triplicate for tests -- no longer doing command line input to pytest --- tests/test_run_ref.py | 203 ++++++++++++++++++++++++------------------ 1 file changed, 117 insertions(+), 86 deletions(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 707f7153..45671f6f 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -16,57 +16,21 @@ pytest.importorskip("mpi4py") # noqa -# TODO: get running for current 8 sims (add necessary data files) -# AND DO THE FIX FOR THE GOOGLE DRIVE (UPLOAD THE FILES AND -# GET THEM TO DOWNLOAD (I GUESS TEMP JUST MAKE ANOTHER -# DICT OR FILE OR DATA STRUCTURE THAT CAN GRAB THE FILES) - -# global list of reference simulation names -# to be used with the dictionary of gdrive file ids in download_sims, -# and to construct yaml and output filenames -ci_ref_sims = [ - "1.1_uniform", - "1.1_gauss", - #"1.1_hera", - #"1.1_gauss", - #"1.2_uniform", - #"1.2_hera", - #"1.3_uniform", - #"1.3_gauss", -] - -# TODO: swap to something more permanent! -# TODO: double confirm downloading is only happening once -@pytest.fixture -def download_sims(): +# TODO: comment method as custom +# method to download sim output -- currently from google drive +# takes as input a directory +def download_sim(target_dir, sim, fid): import requests - target_dir = "results_data" + target_dir = os.path.join(target_dir, "results_data") if not os.path.exists(target_dir): os.makedirs(target_dir) - # temporary way to store all the fileids - # control which simulations via global ci_ref_sims array - fids = { - "1.1_uniform":"1V4RmmUGrx5iH-Zyuj45p1ag3vmMuUI2A", - "1.1_gauss":"1gTj9BSunEoyde5Dq8a5IhUieMo9PuMuC", - #"1.1_hera":"13B0YOSO9v4KsEGX9GVHuBEQrt5Tnjsxr", - #"1.2_gauss":a, - #"1.2_uniform":a, - #"1.2_hera":a, - #"1.3_uniform":a, - #"1.3_gauss":a - } - urlbase = "https://drive.google.com/uc?export=download" - # for each sim name in ci_ref_sims, checks that needs hera uvbeam and - # has downloadable data (is a key to fids) - download_hera_uvbeam = any(["hera" in sim for sim in ci_ref_sims if sim in fids]) - # download the hera uvbeam data file if we need it - if download_hera_uvbeam: + if "hera" in sim: # skipping as file cannot currently be downloaded pass @@ -79,23 +43,41 @@ def download_sims(): # TODO: upload all other files and hardcode # them via a dictionary with ci_ref_sims for keys - for sim in ci_ref_sims: - if sim not in fids.keys(): - continue + # get content + r = requests.get(urlbase, params={"id": fid}) - # get id from dict - fid = fids[sim] - - # get content - r = requests.get(urlbase, params={"id": fid}) + # set filename with full filepath + fname = "ref_" + sim + ".uvh5" + fname = os.path.join(target_dir, fname) - # set filename with full filepath - fname = "ref_" + sim + ".uvh5" - fname = os.path.join(target_dir, fname) + # write the file + with open(fname, "wb") as ofile: + ofile.write(r.content) - # write the file - with open(fname, "wb") as ofile: - ofile.write(r.content) +# TODO +def compare_uvh5(uv_ref, uv_new): + # fix part(s) that should deviate + # TODO: maybe assert that deviation occurred + uv_new.history = uv_ref.history + + # perform equality check between historical and current reference + # simulation output + # TODO: implement better asserts + # include lower tolerance for deviations + assert uv_new == uv_ref + +def construct_filepaths(target_dir, sim): + # construct filename and filepath of downloaded file + uvh5_filename = "ref_" + sim + ".uvh5" + uvh5_filepath = os.path.join(target_dir, "results_data", uvh5_filename) + + # construct filename and filepath of yaml file used + # to run simulation + yaml_filename = "obsparam_ref_" + sim + ".yaml" + yaml_filepath = os.path.join(SIM_DATA_PATH, "test_config", yaml_filename) + assert os.path.exists(yaml_filepath) + + return uvh5_filepath, yaml_filepath @pytest.fixture @@ -109,28 +91,30 @@ def goto_tempdir(tmpdir): os.chdir(cwd) - -# TODO: add all 8 yaml files -@pytest.mark.parametrize("paramfile", ci_ref_sims) +################################################################################ +# Note: I don't have a better approach for the benchmarking so current # +# approach has triplicate test_run_## method for benchmarking workflow # +################################################################################ + +@pytest.mark.parametrize("sim_name, sim_id", + [ + ("1.1_uniform", "1V4RmmUGrx5iH-Zyuj45p1ag3vmMuUI2A"), + ("1.1_gauss", "1gTj9BSunEoyde5Dq8a5IhUieMo9PuMuC"), + ], + ids= ["1.1_uniform", "1.1_gauss"], + ) @pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") -def test_run_11(benchmark, goto_tempdir, download_sims, paramfile): - # download reference sim - # download_sims(goto_tempdir) +def test_run_11(benchmark, goto_tempdir, sim_name, sim_id): + # download reference sim output to compare + download_sim(goto_tempdir, sim_name, sim_id) - # construct filename and filepath of downloaded file - uvh5_filename = "ref_" + paramfile + ".uvh5" - uvh5_filepath = os.path.join(goto_tempdir, "results_data", uvh5_filename) + # construct paths to necessary file using paramfile + uvh5_filepath, yaml_filepath = construct_filepaths(goto_tempdir, sim_name) # instantiate UVData object and read in the downloaded uvh5 # file as the point of historical comparison uv_ref = UVData.from_file(uvh5_filepath) - # construct filename and filepath of yaml file used - # to run simulation - yaml_filename = "obsparam_ref_" + paramfile + ".yaml" - yaml_filepath = os.path.join(SIM_DATA_PATH, "test_config", yaml_filename) - assert os.path.exists(yaml_filepath) - # benchmark uvsim.run_uvsim method with param_filename argument # runs multiple times to get benchmark data # outdir is given by the yaml file but should not write anything @@ -138,25 +122,72 @@ def test_run_11(benchmark, goto_tempdir, download_sims, paramfile): uv_new = benchmark(run_uvsim, yaml_filepath, return_uv=True) # loading the file and comparing is only done on rank 0. + # probably not necessary if pyuvsim.mpi.rank != 0: return - # fix part(s) that should deviate - # TODO: maybe assert that deviation occurred - uv_new.history = uv_ref.history - - # perform equality check between historical and current reference - # simulation output - # TODO: implement better asserts - # include lower tolerance for deviations - assert uv_new == uv_ref + compare_uvh5(uv_ref, uv_new) +@pytest.mark.parametrize("sim_name, sim_id", + [ + ("1.1_uniform", "1V4RmmUGrx5iH-Zyuj45p1ag3vmMuUI2A"), + ("1.1_gauss", "1gTj9BSunEoyde5Dq8a5IhUieMo9PuMuC"), + ], + ids= ["1.2_uniform", "1.2_gauss"], + ) @pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") -def test_run_12(benchmark, goto_tempdir): - output = benchmark(sum, [1,1]) - assert output == 2 +def test_run_12(benchmark, goto_tempdir, sim_name, sim_id): + # download reference sim output to compare + download_sim(goto_tempdir, sim_name, sim_id) + + # construct paths to necessary file using paramfile + uvh5_filepath, yaml_filepath = construct_filepaths(goto_tempdir, sim_name) + + # instantiate UVData object and read in the downloaded uvh5 + # file as the point of historical comparison + uv_ref = UVData.from_file(uvh5_filepath) + + # benchmark uvsim.run_uvsim method with param_filename argument + # runs multiple times to get benchmark data + # outdir is given by the yaml file but should not write anything + # and simply return a UVData object + uv_new = benchmark(run_uvsim, yaml_filepath, return_uv=True) + + # loading the file and comparing is only done on rank 0. + # probably not necessary + if pyuvsim.mpi.rank != 0: + return + compare_uvh5(uv_ref, uv_new) + +@pytest.mark.parametrize("sim_name, sim_id", + [ + ("1.1_uniform", "1V4RmmUGrx5iH-Zyuj45p1ag3vmMuUI2A"), + ("1.1_gauss", "1gTj9BSunEoyde5Dq8a5IhUieMo9PuMuC"), + ], + ids= ["1.3_uniform", "1.3_gauss"], + ) @pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") -def test_run_13(benchmark, goto_tempdir): - output = benchmark(sum, [1,1]) - assert output == 2 +def test_run_13(benchmark, goto_tempdir, sim_name, sim_id): + # download reference sim output to compare + download_sim(goto_tempdir, sim_name, sim_id) + + # construct paths to necessary file using paramfile + uvh5_filepath, yaml_filepath = construct_filepaths(goto_tempdir, sim_name) + + # instantiate UVData object and read in the downloaded uvh5 + # file as the point of historical comparison + uv_ref = UVData.from_file(uvh5_filepath) + + # benchmark uvsim.run_uvsim method with param_filename argument + # runs multiple times to get benchmark data + # outdir is given by the yaml file but should not write anything + # and simply return a UVData object + uv_new = benchmark(run_uvsim, yaml_filepath, return_uv=True) + + # loading the file and comparing is only done on rank 0. + # probably not necessary + if pyuvsim.mpi.rank != 0: + return + + compare_uvh5(uv_ref, uv_new) From 7e0691889730919e15165e1d2bbbd79fdf7f4f25 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 7 Nov 2024 07:40:10 -0500 Subject: [PATCH 39/82] added 1.3 sim files to data --- src/pyuvsim/data/baseline_lite.csv | 6 ++++++ src/pyuvsim/data/bl_lite_gauss.yaml | 6 ++++++ src/pyuvsim/data/bl_lite_uniform.yaml | 5 +++++ .../letter_R_12pt_2458098.38824015.txt | 13 +++++++++++++ .../test_config/obsparam_ref_1.3_gauss.yaml | 18 ++++++++++++++++++ .../test_config/obsparam_ref_1.3_uniform.yaml | 18 ++++++++++++++++++ 6 files changed, 66 insertions(+) create mode 100644 src/pyuvsim/data/baseline_lite.csv create mode 100644 src/pyuvsim/data/bl_lite_gauss.yaml create mode 100644 src/pyuvsim/data/bl_lite_uniform.yaml create mode 100644 src/pyuvsim/data/test_catalogs/letter_R_12pt_2458098.38824015.txt create mode 100644 src/pyuvsim/data/test_config/obsparam_ref_1.3_gauss.yaml create mode 100644 src/pyuvsim/data/test_config/obsparam_ref_1.3_uniform.yaml diff --git a/src/pyuvsim/data/baseline_lite.csv b/src/pyuvsim/data/baseline_lite.csv new file mode 100644 index 00000000..b9cfa199 --- /dev/null +++ b/src/pyuvsim/data/baseline_lite.csv @@ -0,0 +1,6 @@ +Name Number BeamID E N U + +ANT1 0 0 0.0000 0.0000 0.0000 +ANT2 1 0 50.000 0.0000 0.0000 +ANT3 2 0 0.0000 -50.00 0.0000 +ANT4 3 0 26.000 -26.00 0.0000 diff --git a/src/pyuvsim/data/bl_lite_gauss.yaml b/src/pyuvsim/data/bl_lite_gauss.yaml new file mode 100644 index 00000000..8684972a --- /dev/null +++ b/src/pyuvsim/data/bl_lite_gauss.yaml @@ -0,0 +1,6 @@ +beam_paths: + 0 : !AnalyticBeam + class: GaussianBeam + sigma: 0.08449 +telescope_location: (-30.72152777777791, 21.428305555555557, 1073.0000000093132) +telescope_name: BLLITE diff --git a/src/pyuvsim/data/bl_lite_uniform.yaml b/src/pyuvsim/data/bl_lite_uniform.yaml new file mode 100644 index 00000000..d534300b --- /dev/null +++ b/src/pyuvsim/data/bl_lite_uniform.yaml @@ -0,0 +1,5 @@ +beam_paths: + 0 : !AnalyticBeam + class: UniformBeam +telescope_location: (-30.72152777777791, 21.428305555555557, 1073.0000000093132) +telescope_name: BLLITE diff --git a/src/pyuvsim/data/test_catalogs/letter_R_12pt_2458098.38824015.txt b/src/pyuvsim/data/test_catalogs/letter_R_12pt_2458098.38824015.txt new file mode 100644 index 00000000..21bae804 --- /dev/null +++ b/src/pyuvsim/data/test_catalogs/letter_R_12pt_2458098.38824015.txt @@ -0,0 +1,13 @@ +SOURCE_ID RA_ICRS [deg] Dec_ICRS [deg] Flux [Jy] Frequency [Hz] +HERATEST5 59.37045 -28.778843 1 100000000.0 +HERATEST6 57.08925 -28.74223 1 100000000.0 +HERATEST12 59.38125 -27.778828 1 100000000.0 +HERATEST13 58.25100 -27.765359 1 100000000.0 +HERATEST21 59.39115 -26.779049 1 100000000.0 +HERATEST22 58.27125 -26.765736 1 100000000.0 +HERATEST23 57.15150 -26.743624 1 100000000.0 +HERATEST30 59.40120 -25.779269 1 100000000.0 +HERATEST31 57.18090 -25.744495 1 100000000.0 +HERATEST39 59.41035 -24.779242 1 100000000.0 +HERATEST40 58.30965 -24.766704 1 100000000.0 +HERATEST41 57.20820 -24.744905 1 100000000.0 diff --git a/src/pyuvsim/data/test_config/obsparam_ref_1.3_gauss.yaml b/src/pyuvsim/data/test_config/obsparam_ref_1.3_gauss.yaml new file mode 100644 index 00000000..19b0e146 --- /dev/null +++ b/src/pyuvsim/data/test_config/obsparam_ref_1.3_gauss.yaml @@ -0,0 +1,18 @@ +filing: + outdir: '.' + outfile_name: 'ref_1.3_gauss' + output_format: 'uvh5' +freq: + Nfreqs: 6250. + channel_width: 32000.0 + start_freq: 50016000.0 + end_freq: 249984000.0 +sources: + catalog: '../test_catalogs/letter_R_12pt_2458098.38824015.txt' +telescope: + array_layout: '../baseline_lite.csv' + telescope_config_name: '../bl_lite_gauss.yaml' +time: + Ntimes: 2 + integration_time: 11.0 + start_time: 2458098.38824015 diff --git a/src/pyuvsim/data/test_config/obsparam_ref_1.3_uniform.yaml b/src/pyuvsim/data/test_config/obsparam_ref_1.3_uniform.yaml new file mode 100644 index 00000000..a25d0920 --- /dev/null +++ b/src/pyuvsim/data/test_config/obsparam_ref_1.3_uniform.yaml @@ -0,0 +1,18 @@ +filing: + outdir: '.' + outfile_name: 'ref_1.3_uniform' + output_format: 'uvh5' +freq: + Nfreqs: 6250. + channel_width: 32000.0 + start_freq: 50016000.0 + end_freq: 249984000.0 +sources: + catalog: '../test_catalogs/letter_R_12pt_2458098.38824015.txt' +telescope: + array_layout: '../baseline_lite.csv' + telescope_config_name: '../bl_lite_uniform.yaml' +time: + Ntimes: 2 + integration_time: 11.0 + start_time: 2458098.38824015 From a0c11843e37f1f98f6c327bc1541c915918cea1c Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 7 Nov 2024 08:15:58 -0500 Subject: [PATCH 40/82] added 1.2 simulations to data --- .../two_distant_points_2458098.38824015.txt | 3 +++ .../test_config/obsparam_ref_1.2_gauss.yaml | 17 +++++++++++++++++ .../test_config/obsparam_ref_1.2_uniform.yaml | 17 +++++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 src/pyuvsim/data/test_catalogs/two_distant_points_2458098.38824015.txt create mode 100644 src/pyuvsim/data/test_config/obsparam_ref_1.2_gauss.yaml create mode 100644 src/pyuvsim/data/test_config/obsparam_ref_1.2_uniform.yaml diff --git a/src/pyuvsim/data/test_catalogs/two_distant_points_2458098.38824015.txt b/src/pyuvsim/data/test_catalogs/two_distant_points_2458098.38824015.txt new file mode 100644 index 00000000..b0d11243 --- /dev/null +++ b/src/pyuvsim/data/test_catalogs/two_distant_points_2458098.38824015.txt @@ -0,0 +1,3 @@ +SOURCE_ID RA_ICRS [deg] Dec_ICRS [deg] Flux [Jy] Frequency [Hz] +TWOPOINT0 108.05159 -0.9894784 1 100000000.0 +TWOPOINT1 291.49730 -1.0600652 1 100000000.0 diff --git a/src/pyuvsim/data/test_config/obsparam_ref_1.2_gauss.yaml b/src/pyuvsim/data/test_config/obsparam_ref_1.2_gauss.yaml new file mode 100644 index 00000000..5ae208bc --- /dev/null +++ b/src/pyuvsim/data/test_config/obsparam_ref_1.2_gauss.yaml @@ -0,0 +1,17 @@ +filing: + outdir: '.' + outfile_name: 'ref_1.2_gauss' + output_format: 'uvh5' +freq: + Nfreqs: 1 + channel_width: 80000.0 + start_freq: 100000000.0 +sources: + catalog: '../test_catalogs/two_distant_points_2458098.38824015.txt' +telescope: + array_layout: '../baseline_lite.csv' + telescope_config_name: '../bl_lite_gauss.yaml' +time: + Ntimes: 86400 + integration_time: 1.0 + start_time: 2458098.38824015 diff --git a/src/pyuvsim/data/test_config/obsparam_ref_1.2_uniform.yaml b/src/pyuvsim/data/test_config/obsparam_ref_1.2_uniform.yaml new file mode 100644 index 00000000..4b374fff --- /dev/null +++ b/src/pyuvsim/data/test_config/obsparam_ref_1.2_uniform.yaml @@ -0,0 +1,17 @@ +filing: + outdir: '.' + outfile_name: 'ref_1.2_uniform' + output_format: 'uvh5' +freq: + Nfreqs: 1 + channel_width: 80000.0 + start_freq: 100000000.0 +sources: + catalog: '../test_catalogs/two_distant_points_2458098.38824015.txt' +telescope: + array_layout: '../baseline_lite.csv' + telescope_config_name: '../bl_lite_uniform.yaml' +time: + Ntimes: 86400 + integration_time: 1.0 + start_time: 2458098.38824015 From f38ea1c4e1783e0583452c7acf43b369cafb5e53 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 7 Nov 2024 08:46:45 -0500 Subject: [PATCH 41/82] google drive links --- tests/test_run_ref.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 45671f6f..64767fc9 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -130,8 +130,8 @@ def test_run_11(benchmark, goto_tempdir, sim_name, sim_id): @pytest.mark.parametrize("sim_name, sim_id", [ - ("1.1_uniform", "1V4RmmUGrx5iH-Zyuj45p1ag3vmMuUI2A"), - ("1.1_gauss", "1gTj9BSunEoyde5Dq8a5IhUieMo9PuMuC"), + ("1.2_uniform", "1Qo7Mu6AX2_1OcV0f88YwOPwpqHELIEgi"), + ("1.2_gauss", "1a9ZS189c7c-a2heE54Kdm6Kl3_6EKNSo"), ], ids= ["1.2_uniform", "1.2_gauss"], ) @@ -162,8 +162,8 @@ def test_run_12(benchmark, goto_tempdir, sim_name, sim_id): @pytest.mark.parametrize("sim_name, sim_id", [ - ("1.1_uniform", "1V4RmmUGrx5iH-Zyuj45p1ag3vmMuUI2A"), - ("1.1_gauss", "1gTj9BSunEoyde5Dq8a5IhUieMo9PuMuC"), + ("1.3_uniform", "1XRTEGtmdw3MTpfiw-pMAyjoVBxq-rDIs"), + ("1.3_gauss", "1qzigv8BAHhQ6-Y9Au3TlwSHoc6RqFdii"), ], ids= ["1.3_uniform", "1.3_gauss"], ) From 58bb1498a58f94211bd3ea551b20f241719a4367 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 7 Nov 2024 09:02:25 -0500 Subject: [PATCH 42/82] swapped workflow simulation run order --- .github/workflows/testsuite.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index a4ff680a..e2c765d4 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -224,10 +224,10 @@ jobs: include: - id: test_run_11 name: "1.1 Reference Simulations" - - id: test_run_12 - name: "1.2 Reference Simulations" - id: test_run_13 name: "1.3 Reference Simulations" + - id: test_run_12 + name: "1.2 Reference Simulations" defaults: run: From 2aa4a975c2ae47d388ad7bac6bd44a567fecf794 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Fri, 15 Nov 2024 15:57:45 -0500 Subject: [PATCH 43/82] Swapped to downloading files from the BDR -- will trim workflow runtime down to ~1 hour using pedantic. Added mwa uvbeam sim files to data not yet tested. --- src/pyuvsim/data/bl_lite_mwa.yaml | 6 ++ src/pyuvsim/data/mwa88_nocore_config_MWA.yaml | 6 ++ .../test_config/obsparam_ref_1.1_mwa.yaml | 20 +++++ .../test_config/obsparam_ref_1.2_gauss.yaml | 3 + .../test_config/obsparam_ref_1.2_mwa.yaml | 20 +++++ .../test_config/obsparam_ref_1.2_uniform.yaml | 3 + .../test_config/obsparam_ref_1.3_gauss.yaml | 3 + .../test_config/obsparam_ref_1.3_uniform.yaml | 3 + tests/test_run_ref.py | 74 +++++++++++-------- 9 files changed, 109 insertions(+), 29 deletions(-) create mode 100644 src/pyuvsim/data/bl_lite_mwa.yaml create mode 100644 src/pyuvsim/data/mwa88_nocore_config_MWA.yaml create mode 100644 src/pyuvsim/data/test_config/obsparam_ref_1.1_mwa.yaml create mode 100644 src/pyuvsim/data/test_config/obsparam_ref_1.2_mwa.yaml diff --git a/src/pyuvsim/data/bl_lite_mwa.yaml b/src/pyuvsim/data/bl_lite_mwa.yaml new file mode 100644 index 00000000..84b71fb8 --- /dev/null +++ b/src/pyuvsim/data/bl_lite_mwa.yaml @@ -0,0 +1,6 @@ +beam_paths: + 0: !UVBeam + filename: ../data/mwa_full_embedded_element_pattern.h5 + pixels_per_deg: 1 +telescope_location: (-30.72152777777791, 21.428305555555557, 1073.0000000093132) +telescope_name: BLLITE diff --git a/src/pyuvsim/data/mwa88_nocore_config_MWA.yaml b/src/pyuvsim/data/mwa88_nocore_config_MWA.yaml new file mode 100644 index 00000000..f90d0c68 --- /dev/null +++ b/src/pyuvsim/data/mwa88_nocore_config_MWA.yaml @@ -0,0 +1,6 @@ +beam_paths: + 0: !UVBeam + filename: ../data/mwa_full_embedded_element_pattern.h5 + pixels_per_deg: 1 +telescope_location: (-30.72152777777791, 21.428305555555557, 1073.0000000093132) +telescope_name: MWA diff --git a/src/pyuvsim/data/test_config/obsparam_ref_1.1_mwa.yaml b/src/pyuvsim/data/test_config/obsparam_ref_1.1_mwa.yaml new file mode 100644 index 00000000..a42baef0 --- /dev/null +++ b/src/pyuvsim/data/test_config/obsparam_ref_1.1_mwa.yaml @@ -0,0 +1,20 @@ +filing: + outdir: '.' + outfile_name: 'ref_1.1_mwa' + output_format: 'uvh5' +freq: + Nfreqs: 1 + channel_width: 80000.0 + start_freq: 100000000.0 +sources: + catalog: '../test_catalogs/mock_catalog_heratext_2458098.38824015.txt' +telescope: + array_layout: '../mwa_nocore_layout.csv' + telescope_config_name: '../mwa88_nocore_config_MWA.yaml' +time: + Ntimes: 1 + integration_time: 11.0 + start_time: 2458098.38824015 +ordering: + conjugation_convention: ant1 3 workers), and simply have 1.2 run maybe twice each?, 1.1 / 1.3 unedited (if want faster just make 1.2 run once each I guess) + +# NOTE: current approach should still work unless malicious actors do something lol + @pytest.mark.parametrize("sim_name, sim_id", [ ("1.1_uniform", "1V4RmmUGrx5iH-Zyuj45p1ag3vmMuUI2A"), @@ -106,7 +122,7 @@ def goto_tempdir(tmpdir): @pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") def test_run_11(benchmark, goto_tempdir, sim_name, sim_id): # download reference sim output to compare - download_sim(goto_tempdir, sim_name, sim_id) + download_sim(goto_tempdir, sim_name) # construct paths to necessary file using paramfile uvh5_filepath, yaml_filepath = construct_filepaths(goto_tempdir, sim_name) @@ -138,7 +154,7 @@ def test_run_11(benchmark, goto_tempdir, sim_name, sim_id): @pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") def test_run_12(benchmark, goto_tempdir, sim_name, sim_id): # download reference sim output to compare - download_sim(goto_tempdir, sim_name, sim_id) + download_sim(goto_tempdir, sim_name) # construct paths to necessary file using paramfile uvh5_filepath, yaml_filepath = construct_filepaths(goto_tempdir, sim_name) @@ -170,7 +186,7 @@ def test_run_12(benchmark, goto_tempdir, sim_name, sim_id): @pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") def test_run_13(benchmark, goto_tempdir, sim_name, sim_id): # download reference sim output to compare - download_sim(goto_tempdir, sim_name, sim_id) + download_sim(goto_tempdir, sim_name) # construct paths to necessary file using paramfile uvh5_filepath, yaml_filepath = construct_filepaths(goto_tempdir, sim_name) From 861d83d94dc1f6db9c4e521fdf1180f3f53a7605 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 18 Nov 2024 15:56:35 -0500 Subject: [PATCH 44/82] figured out approach to parametrize all reference simulation tests using workflow. Still need to integrate mwa sims. Added pedantic benchmarking. Need to determine best approach to setting up workflow matrix. --- .github/workflows/testsuite.yaml | 20 +++--- tests/conftest.py | 8 +++ tests/test_run_ref.py | 103 +++++++++---------------------- 3 files changed, 47 insertions(+), 84 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index e2c765d4..63ce3b7b 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -209,7 +209,9 @@ jobs: env_vars: OS,PYTHON fail_ci_if_error: true - # TODO: should implement generic method of determining os instead of hardcoding it + # TODO: implement benchmarking that doesn't rely on caching if possible -- probably via pushing to another git repository or similar + # TODO: Redo matrix using new pytest test with refsim command line argument so it runs an independent environment for each ref sim + # NOTE: can technically improve this by grouping reference sims by generation in the matrix and then have them be parametrized in batches! (DO THIS!) benchmark: name: Performance Benchmark needs: tests @@ -219,15 +221,15 @@ jobs: runs-on: ubuntu-latest strategy: - max-parallel: 1 + #max-parallel: 1 matrix: include: - - id: test_run_11 - name: "1.1 Reference Simulations" - - id: test_run_13 - name: "1.3 Reference Simulations" - - id: test_run_12 - name: "1.2 Reference Simulations" + - id: 1.1_uniform + name: "Reference Simulation 1.1 uniform" + - id: 1.1_gauss + name: "Reference Simulation 1.1 gauss" + - id: 1.2_uniform + name: "Reference Simulation 1.2 uniform" defaults: run: @@ -263,7 +265,7 @@ jobs: - name: Run benchmark run: | - mpiexec -n 1 -np 1 pytest -k ${{ matrix.id }} --benchmark-only --benchmark-json output.json + mpiexec -n 1 -np 1 pytest --refsim=${{ matrix.id }} --benchmark-only --benchmark-json output.json -s # TODO: figure out why / if okay to delete before downloading? - name: Clear previous benchmark report diff --git a/tests/conftest.py b/tests/conftest.py index 93398dbf..06cb039f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -62,6 +62,14 @@ def pytest_addoption(parser): parser.addoption( "--nompi", action="store_true", help="skip mpi-parallelized tests." ) + parser.addoption( + "--refsim", action="append", default=[], + help="list of refsim names to pass to test functions." + ) + +def pytest_generate_tests(metafunc): + if "refsim" in metafunc.fixturenames: + metafunc.parametrize("refsim", metafunc.config.getoption("refsim")) def pytest_runtest_setup(item): diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 8fe25424..f2cbabf3 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -100,96 +100,36 @@ def goto_tempdir(tmpdir): os.chdir(cwd) +#def test_run_sim(refsim, goto_tempdir): +# print("\n\n" + refsim + "\n\n") +# assert refsim == "test" + +#ids= ["1.1_uniform", "1.1_gauss"], +#ids= ["1.2_uniform", "1.2_gauss"], +#ids= ["1.3_uniform", "1.3_gauss"], + ################################################################################ # Note: I don't have a better approach for the benchmarking so current # # approach has triplicate test_run_## method for benchmarking workflow # ################################################################################ -# TODO: FIXME: GET RID OF GOOGLE DRIVE IDS IN PARAMETRIZE! # TODO: FIXME: IMPLEMENT MWA BEAM DOWNLOADING AND THE RUNNING OF THE MWA 1.1 AND 1.2 SIMULATIONS # TODO: FIXME: (TECHNICALLY GOES ABOVE BUT HAVE IT EXPLICITLY SEARCH COLLECTION AND NOT GENERAL SEARCH API FOR BETTER RESULTS) # TODO: FIXME: customize 1.2 and maybe 1.3 to have reduced iterations (hopefully fluctuation isn't that bad). Shoot for 1 hour total runtime with all tests, so probably stop forcing order for reference sims (assuming > 3 workers), and simply have 1.2 run maybe twice each?, 1.1 / 1.3 unedited (if want faster just make 1.2 run once each I guess) # NOTE: current approach should still work unless malicious actors do something lol -@pytest.mark.parametrize("sim_name, sim_id", - [ - ("1.1_uniform", "1V4RmmUGrx5iH-Zyuj45p1ag3vmMuUI2A"), - ("1.1_gauss", "1gTj9BSunEoyde5Dq8a5IhUieMo9PuMuC"), - ], - ids= ["1.1_uniform", "1.1_gauss"], - ) -@pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") -def test_run_11(benchmark, goto_tempdir, sim_name, sim_id): - # download reference sim output to compare - download_sim(goto_tempdir, sim_name) - - # construct paths to necessary file using paramfile - uvh5_filepath, yaml_filepath = construct_filepaths(goto_tempdir, sim_name) - - # instantiate UVData object and read in the downloaded uvh5 - # file as the point of historical comparison - uv_ref = UVData.from_file(uvh5_filepath) - - # benchmark uvsim.run_uvsim method with param_filename argument - # runs multiple times to get benchmark data - # outdir is given by the yaml file but should not write anything - # and simply return a UVData object - uv_new = benchmark(run_uvsim, yaml_filepath, return_uv=True) - - # loading the file and comparing is only done on rank 0. - # probably not necessary - if pyuvsim.mpi.rank != 0: - return - - compare_uvh5(uv_ref, uv_new) - -@pytest.mark.parametrize("sim_name, sim_id", - [ - ("1.2_uniform", "1Qo7Mu6AX2_1OcV0f88YwOPwpqHELIEgi"), - ("1.2_gauss", "1a9ZS189c7c-a2heE54Kdm6Kl3_6EKNSo"), - ], - ids= ["1.2_uniform", "1.2_gauss"], - ) -@pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") -def test_run_12(benchmark, goto_tempdir, sim_name, sim_id): - # download reference sim output to compare - download_sim(goto_tempdir, sim_name) - - # construct paths to necessary file using paramfile - uvh5_filepath, yaml_filepath = construct_filepaths(goto_tempdir, sim_name) - - # instantiate UVData object and read in the downloaded uvh5 - # file as the point of historical comparison - uv_ref = UVData.from_file(uvh5_filepath) - - # benchmark uvsim.run_uvsim method with param_filename argument - # runs multiple times to get benchmark data - # outdir is given by the yaml file but should not write anything - # and simply return a UVData object - uv_new = benchmark(run_uvsim, yaml_filepath, return_uv=True) - - # loading the file and comparing is only done on rank 0. - # probably not necessary - if pyuvsim.mpi.rank != 0: - return - - compare_uvh5(uv_ref, uv_new) +# list of sims to benchmark pedantically as they are too long to run many times +# TODO: if need be swap to a dictionary with more specific custom pedantic arguments +long_ref_sims = ["1.2_uniform", "1.2_gauss"] -@pytest.mark.parametrize("sim_name, sim_id", - [ - ("1.3_uniform", "1XRTEGtmdw3MTpfiw-pMAyjoVBxq-rDIs"), - ("1.3_gauss", "1qzigv8BAHhQ6-Y9Au3TlwSHoc6RqFdii"), - ], - ids= ["1.3_uniform", "1.3_gauss"], - ) @pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") -def test_run_13(benchmark, goto_tempdir, sim_name, sim_id): +def test_run_sim(benchmark, goto_tempdir, refsim): # download reference sim output to compare - download_sim(goto_tempdir, sim_name) + download_sim(goto_tempdir, refsim) # construct paths to necessary file using paramfile - uvh5_filepath, yaml_filepath = construct_filepaths(goto_tempdir, sim_name) + uvh5_filepath, yaml_filepath = construct_filepaths(goto_tempdir, refsim) # instantiate UVData object and read in the downloaded uvh5 # file as the point of historical comparison @@ -199,7 +139,20 @@ def test_run_13(benchmark, goto_tempdir, sim_name, sim_id): # runs multiple times to get benchmark data # outdir is given by the yaml file but should not write anything # and simply return a UVData object - uv_new = benchmark(run_uvsim, yaml_filepath, return_uv=True) + # uses long_ref_sims global array to determine if pedantic benchmarking should be used + if refsim in long_ref_sims: + print("\n\n" + "pedantic" + "\n\n") + uv_new = benchmark.pedantic( + run_uvsim, + args=[yaml_filepath], + kwargs={'return_uv':True}, + setup=None, + rounds=1, + warmup_rounds=0, + iterations=1) + else: + print("\n\n" + "automatic" + "\n\n") + uv_new = benchmark(run_uvsim, yaml_filepath, return_uv=True) # loading the file and comparing is only done on rank 0. # probably not necessary From e0aabd57425f61510898d06523fb973c93c997c3 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 18 Nov 2024 20:27:41 -0500 Subject: [PATCH 45/82] filled out workflow refsim names, and added 1.1_mwa reference simulation to pytest and workflow. --- .github/workflows/testsuite.yaml | 7 ++-- src/pyuvsim/data/mwa88_nocore_config_MWA.yaml | 5 +-- tests/test_run_ref.py | 34 ++++++++----------- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index 63ce3b7b..ad34059e 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -225,11 +225,12 @@ jobs: matrix: include: - id: 1.1_uniform - name: "Reference Simulation 1.1 uniform" + - id: 1.1_mwa - id: 1.1_gauss - name: "Reference Simulation 1.1 gauss" - id: 1.2_uniform - name: "Reference Simulation 1.2 uniform" + - id: 1.2_gauss + - id: 1.3_uniform + - id: 1.3_gauss defaults: run: diff --git a/src/pyuvsim/data/mwa88_nocore_config_MWA.yaml b/src/pyuvsim/data/mwa88_nocore_config_MWA.yaml index f90d0c68..75921c5d 100644 --- a/src/pyuvsim/data/mwa88_nocore_config_MWA.yaml +++ b/src/pyuvsim/data/mwa88_nocore_config_MWA.yaml @@ -1,6 +1,7 @@ beam_paths: 0: !UVBeam - filename: ../data/mwa_full_embedded_element_pattern.h5 - pixels_per_deg: 1 + filename: mwa_full_embedded_element_pattern.h5 + path_variable: pyuvsim.data.DATA_PATH + pixels_per_deg: 1 telescope_location: (-30.72152777777791, 21.428305555555557, 1073.0000000093132) telescope_name: MWA diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index f2cbabf3..27938ff7 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -35,9 +35,8 @@ def get_latest_pid(response): return latest_pid + # TODO: fix up order of operations and comment the method better -# method to download sim output -- currently from google drive -# takes as input a directory def download_sim(target_dir, sim_name): import requests import urllib.request @@ -62,6 +61,18 @@ def download_sim(target_dir, sim_name): # download the file to the location urllib.request.urlretrieve(download_url, fname) + # additionally download mwa uvbeam to SIM_DATA_PATH if necessary + if "mwa" in sim_name: + download_url = "http://ws.mwatelescope.org/static/mwa_full_embedded_element_pattern.h5" + fname=os.path.join(SIM_DATA_PATH,"mwa_full_embedded_element_pattern.h5") + print(fname) + + # download the file to the location + if os.path.isfile(fname): + print("skipping download mwa uvbeam file already downloaded") + else: + urllib.request.urlretrieve(download_url, fname) + # TODO: make more complete def compare_uvh5(uv_ref, uv_new): @@ -100,24 +111,9 @@ def goto_tempdir(tmpdir): os.chdir(cwd) -#def test_run_sim(refsim, goto_tempdir): -# print("\n\n" + refsim + "\n\n") -# assert refsim == "test" -#ids= ["1.1_uniform", "1.1_gauss"], -#ids= ["1.2_uniform", "1.2_gauss"], -#ids= ["1.3_uniform", "1.3_gauss"], - -################################################################################ -# Note: I don't have a better approach for the benchmarking so current # -# approach has triplicate test_run_## method for benchmarking workflow # -################################################################################ - -# TODO: FIXME: IMPLEMENT MWA BEAM DOWNLOADING AND THE RUNNING OF THE MWA 1.1 AND 1.2 SIMULATIONS +# TODO: FIXME: ASK WHY MWA 1.2 BEAM HARD TO RUN? # TODO: FIXME: (TECHNICALLY GOES ABOVE BUT HAVE IT EXPLICITLY SEARCH COLLECTION AND NOT GENERAL SEARCH API FOR BETTER RESULTS) -# TODO: FIXME: customize 1.2 and maybe 1.3 to have reduced iterations (hopefully fluctuation isn't that bad). Shoot for 1 hour total runtime with all tests, so probably stop forcing order for reference sims (assuming > 3 workers), and simply have 1.2 run maybe twice each?, 1.1 / 1.3 unedited (if want faster just make 1.2 run once each I guess) - -# NOTE: current approach should still work unless malicious actors do something lol # list of sims to benchmark pedantically as they are too long to run many times # TODO: if need be swap to a dictionary with more specific custom pedantic arguments @@ -141,7 +137,6 @@ def test_run_sim(benchmark, goto_tempdir, refsim): # and simply return a UVData object # uses long_ref_sims global array to determine if pedantic benchmarking should be used if refsim in long_ref_sims: - print("\n\n" + "pedantic" + "\n\n") uv_new = benchmark.pedantic( run_uvsim, args=[yaml_filepath], @@ -151,7 +146,6 @@ def test_run_sim(benchmark, goto_tempdir, refsim): warmup_rounds=0, iterations=1) else: - print("\n\n" + "automatic" + "\n\n") uv_new = benchmark(run_uvsim, yaml_filepath, return_uv=True) # loading the file and comparing is only done on rank 0. From e4c3ee822d8ef8e0ed71410d1d946507b91725bb Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Tue, 19 Nov 2024 12:35:27 -0500 Subject: [PATCH 46/82] changed the Brown Digital Repository file downloading to use a collection approach, added some print statements, added a line for formatting --- tests/conftest.py | 1 + tests/test_run_ref.py | 52 +++++++++++++++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 06cb039f..d36dca0b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -67,6 +67,7 @@ def pytest_addoption(parser): help="list of refsim names to pass to test functions." ) + def pytest_generate_tests(metafunc): if "refsim" in metafunc.fixturenames: metafunc.parametrize("refsim", metafunc.config.getoption("refsim")) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 27938ff7..747f00f1 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -16,22 +16,44 @@ pytest.importorskip("mpi4py") # noqa +#TODO: make downloading simulation output file and companion files more robust (add retries, etc...) + # gets latest pid from api call def get_latest_pid(response): - response_content_arr = response.json()['response']['docs'] + import requests - # construct an array of just the timestamps for object creation from all the search results - time_arr = [item['object_created_dsi'] for item in response_content_arr] + # list of item dicts + collection_response_items = response.json()['items']['docs'] - if len(time_arr) == 0: - print("fail") - return + if len(collection_response_items) == 0: + return "No items found in collection matching query." + + # using "object_created_dsi" key to sort the items, so need to request that + # for each item via the "json_uri", and get the pid as well to return + print(f"requesting json_uri for each item in reponse, and parsing 'object_created_dsi' and 'pid'") + json_uris = [ item['json_uri'] for item in collection_response_items ] + object_created_dsis = [] + pids = [] + for uri in json_uris: + # get response for item as json + response_json = requests.get(uri).json() + + # get values from json + time_created = response_json['object_created_dsi'] + item_pid = response_json['pid'] + + # append to lists + object_created_dsis.append(time_created) + pids.append(item_pid) # get the max timestamp, then get the index at which the max time occurs # this corresponds to the latest created object - latest_item_pos = time_arr.index(max(time_arr)) - # get the pid of the latest item - latest_pid = response_content_arr[latest_item_pos]['pid'] + latest_item_pos = object_created_dsis.index(max(object_created_dsis)) + + # get the pid of the latest item by shared pos + latest_pid = pids[latest_item_pos] + + print("returning pid of most recent file uploaded to the collection matching the query") return latest_pid @@ -41,11 +63,18 @@ def download_sim(target_dir, sim_name): import requests import urllib.request - api_url="https://repository.library.brown.edu/api/search/?q=" + api_url="https://repository.library.brown.edu/api/collections/bdr:wte2qah8/?q=" + + print(f"\nquerying BDR collection for items matching {sim_name} via api: {api_url+sim_name}") response=requests.get(api_url+sim_name) + # parse out the latest file in the collection from the search result and return its pid + pid = get_latest_pid(response) + + # append results_data to the target directory download path target_dir = os.path.join(target_dir, "results_data") + # check if we need to make target_dir if not os.path.exists(target_dir): os.makedirs(target_dir) @@ -53,11 +82,10 @@ def download_sim(target_dir, sim_name): fname = "ref_" + sim_name + ".uvh5" fname = os.path.join(target_dir, fname) - pid = get_latest_pid(response) - # download url download_url = f"https://repository.library.brown.edu/storage/{pid}/content/" + print(f"attempting download of requested file by http: {download_url}\n") # download the file to the location urllib.request.urlretrieve(download_url, fname) From 8c1ecb3c9f5f91bb9300e9ff7b2473fc4958b844 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Tue, 19 Nov 2024 15:05:52 -0500 Subject: [PATCH 47/82] removed 1.2_mwa files, minor comments change --- .github/workflows/testsuite.yaml | 9 +++------ src/pyuvsim/data/bl_lite_mwa.yaml | 6 ------ .../test_config/obsparam_ref_1.2_mwa.yaml | 20 ------------------- tests/test_run_ref.py | 4 +--- 4 files changed, 4 insertions(+), 35 deletions(-) delete mode 100644 src/pyuvsim/data/bl_lite_mwa.yaml delete mode 100644 src/pyuvsim/data/test_config/obsparam_ref_1.2_mwa.yaml diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index ad34059e..d7fdf759 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -209,9 +209,8 @@ jobs: env_vars: OS,PYTHON fail_ci_if_error: true - # TODO: implement benchmarking that doesn't rely on caching if possible -- probably via pushing to another git repository or similar - # TODO: Redo matrix using new pytest test with refsim command line argument so it runs an independent environment for each ref sim - # NOTE: can technically improve this by grouping reference sims by generation in the matrix and then have them be parametrized in batches! (DO THIS!) + # TODO: implement benchmarking that doesn't rely on caching -- probably via pushing to another git repository or similar + # TODO: can technically improve this by grouping reference sims by generation in the matrix and then have them be parametrized in batches! (DO THIS!) (not sure how to nicely do this) benchmark: name: Performance Benchmark needs: tests @@ -285,9 +284,7 @@ jobs: key: ${{ matrix.id }}-benchmark # TODO: figure out if this appropriately updates the cache - # 1 - # this step also EDITS the ./cache/benchmark-data.json file - # We do not need to add output.json to the cache directory + # TODO: more research on this - name: Compare benchmarks uses: benchmark-action/github-action-benchmark@v1 with: diff --git a/src/pyuvsim/data/bl_lite_mwa.yaml b/src/pyuvsim/data/bl_lite_mwa.yaml deleted file mode 100644 index 84b71fb8..00000000 --- a/src/pyuvsim/data/bl_lite_mwa.yaml +++ /dev/null @@ -1,6 +0,0 @@ -beam_paths: - 0: !UVBeam - filename: ../data/mwa_full_embedded_element_pattern.h5 - pixels_per_deg: 1 -telescope_location: (-30.72152777777791, 21.428305555555557, 1073.0000000093132) -telescope_name: BLLITE diff --git a/src/pyuvsim/data/test_config/obsparam_ref_1.2_mwa.yaml b/src/pyuvsim/data/test_config/obsparam_ref_1.2_mwa.yaml deleted file mode 100644 index 2da2c4a1..00000000 --- a/src/pyuvsim/data/test_config/obsparam_ref_1.2_mwa.yaml +++ /dev/null @@ -1,20 +0,0 @@ -filing: - outdir: '.' - outfile_name: 'ref_1.2_mwa' - output_format: 'uvh5' -freq: - Nfreqs: 1 - channel_width: 80000.0 - start_freq: 100000000.0 -sources: - catalog: '../test_catalogs/two_distant_points_2458098.38824015.txt' -telescope: - array_layout: '../baseline_lite.csv' - telescope_config_name: '../bl_lite_mwa.yaml' -time: - Ntimes: 86400 - integration_time: 1.0 - start_time: 2458098.38824015 -ordering: - conjugation_convention: ant1 Date: Mon, 25 Nov 2024 06:49:08 -0500 Subject: [PATCH 48/82] Intermediate commit while attempting to switch benchmarking approach to use artifacts. Need to re-integrate the benchmark action, and create structure for concatenating benchmark output and uploading it. --- .github/workflows/compare_post_benchmark.yaml | 131 ++++++++++++++++++ .github/workflows/testsuite.yaml | 97 ------------- 2 files changed, 131 insertions(+), 97 deletions(-) create mode 100644 .github/workflows/compare_post_benchmark.yaml diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml new file mode 100644 index 00000000..69f63c10 --- /dev/null +++ b/.github/workflows/compare_post_benchmark.yaml @@ -0,0 +1,131 @@ +#TODO: configure run on benchmark action completion! +#TODO: setup +#TODO: load artifacts +#TODO: collate artifacts (for now skip!) +#TODO: SET UP TO RUN AFTER THE BENCHMARKING IS DONE (FOLLOW YEW GUIDE!) + +name: "Post benchmark results" + +on: +# replace this after other stuff starts functioning +# workflow_run: +# workflows: ["Tests"] +# types: +# - completed +on: [push, pull_request] + +jobs: + # TODO: implement benchmarking that doesn't rely on caching -- probably via pushing to another git repository or similar + # TODO: can technically improve this by grouping reference sims by generation in the matrix and then have them be parametrized in batches! (DO THIS!) (not sure how to nicely do this) + # TODO: swap to saving artifact for each sim run with slightly custom json (?) follow yew + # TODO: write script that concatenates the json output of multiple benchmark runs! (get this working first) + # make artifacts path for safety reasons + # configure download of artifacts according to the actions and have them be in the one path with merge + # specify the downloading "name" using pattern (for now just *json or something) + # (obviously add this to the other yaml file for post benchmark runs, then once we have the post-benchmark run outputing the found artifacts and this benchmark successfully saving the artifacts (and clarified the uniqueness of artifacts across commits! because idk how that is kept separate and if it isn't separate need to delete / avoid that issue!)) + # then can go back to concatenation (or figure out how best to simplify that / scriptwise!) + # then can work on the github-action-benchmark on gh-pages and see how that looks once it's all concatenated, and determine if we need the write permissions + # note: need to separate the push attempt and do so only if we are pushing to "main" ie (I THINK) autopush: false, all else should be comfortable + # if this works then gh-pages only updated with a push when it is to "main", and all else will just compare hopefully, I THINK I CAN GET AWAY WITH DOING "autopush: (fix syntax here) branch == "main"" + benchmark: + if: github.event.workflow_run.conclusion == 'success' + name: Performance Benchmark + env: + ENV_NAME: pyuvsim_tests_mpich + PYTHON: "3.12" + runs-on: ubuntu-latest + + strategy: + #max-parallel: 1 + matrix: + include: + - id: 1.1_uniform + - id: 1.1_gauss +# - id: 1.1_mwa +# - id: 1.2_uniform +# - id: 1.2_gauss +# - id: 1.3_uniform +# - id: 1.3_gauss + + defaults: + run: + # Adding -l {0} helps ensure conda can be found properly. + shell: bash -l {0} + steps: + - uses: actions/checkout@main + + - name: Setup Miniforge + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-version: latest + python-version: ${{ env.PYTHON }} + environment-file: ci/${{ env.ENV_NAME }}.yaml + activate-environment: ${{ env.ENV_NAME }} + run-post: false + + - name: Conda Info + run: | + conda info -a + conda list + PYVER=`python -c "import sys; print('{:d}.{:d}'.format(sys.version_info.major, sys.version_info.minor))"` + if [[ $PYVER != $PYTHON ]]; then + exit 1; + fi + + # also install benchmark utility and requests + - name: Install + run: | + pip install pytest-benchmark + pip install requests + pip install . + + - name: Run benchmark + run: | + # TODO: determine if can specify a directory for file output if directory exists (if so delete third line in ez modification) + mkdir artifacts/ + mpiexec -n 1 -np 1 pytest --refsim=${{ matrix.id }} --benchmark-only --benchmark-json artifacts/output_${{ matrix.id }}.json -s + + - name: Upload result artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }} + path: artifacts/ + if-no-files-found: error + include-hidden-files: true + retention-days: 1 + + collate-post-benchmark: + name: Concatenate and Post Benchmark Results + needs: benchmark + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + + steps: + # Checkout repo for the github-action-benchmark action + - uses: actions/checkout@v4 + + # should only download from current workflow + - name: Download result artifacts + uses: actions/download-artifact@v3 + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + pattern: ${{ github.run_id }}-${{ github.run_attempt }}-* + merge-multiple: true + path: artifacts + - name: Display structure of downloaded files + run: ls -R + + + +# TODO: swap to be pytest and the works +#- name: Store benchmark result +# uses: benchmark-action/github-action-benchmark@v1 +# with: +# name: My Project Go Benchmark +# tool: 'go' +# output-file-path: output.txt +# # Set auto-push to false since GitHub API token is not given +# auto-push: false # todo put in logic if branch == main && push or something! +# Push gh-pages branch by yourself +#- name: Push benchmark result +# run: git push 'https://you:${{ secrets.GITHUB_TOKEN }}@github.com/you/repo-name.git' gh-pages:gh-pages diff --git a/.github/workflows/testsuite.yaml b/.github/workflows/testsuite.yaml index d7fdf759..be5adfff 100644 --- a/.github/workflows/testsuite.yaml +++ b/.github/workflows/testsuite.yaml @@ -208,100 +208,3 @@ jobs: file: ./coverage.xml #optional env_vars: OS,PYTHON fail_ci_if_error: true - - # TODO: implement benchmarking that doesn't rely on caching -- probably via pushing to another git repository or similar - # TODO: can technically improve this by grouping reference sims by generation in the matrix and then have them be parametrized in batches! (DO THIS!) (not sure how to nicely do this) - benchmark: - name: Performance Benchmark - needs: tests - env: - ENV_NAME: pyuvsim_tests_mpich - PYTHON: "3.12" - runs-on: ubuntu-latest - - strategy: - #max-parallel: 1 - matrix: - include: - - id: 1.1_uniform - - id: 1.1_mwa - - id: 1.1_gauss - - id: 1.2_uniform - - id: 1.2_gauss - - id: 1.3_uniform - - id: 1.3_gauss - - defaults: - run: - # Adding -l {0} helps ensure conda can be found properly. - shell: bash -l {0} - steps: - - uses: actions/checkout@main - - - name: Setup Miniforge - uses: conda-incubator/setup-miniconda@v3 - with: - miniforge-version: latest - python-version: ${{ env.PYTHON }} - environment-file: ci/${{ env.ENV_NAME }}.yaml - activate-environment: ${{ env.ENV_NAME }} - run-post: false - - - name: Conda Info - run: | - conda info -a - conda list - PYVER=`python -c "import sys; print('{:d}.{:d}'.format(sys.version_info.major, sys.version_info.minor))"` - if [[ $PYVER != $PYTHON ]]; then - exit 1; - fi - - # also install benchmark utility and requests - - name: Install - run: | - pip install pytest-benchmark - pip install requests - pip install . - - - name: Run benchmark - run: | - mpiexec -n 1 -np 1 pytest --refsim=${{ matrix.id }} --benchmark-only --benchmark-json output.json -s - - # TODO: figure out why / if okay to delete before downloading? - - name: Clear previous benchmark report - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/1.1_ref_sim_ci_workflow'}} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh cache delete ${{ matrix.id }}-benchmark --repo RadioAstronomySoftwareGroup/pyuvsim - continue-on-error: true - - # Download previous benchmark result from cache (if exists) - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ matrix.id }}-benchmark - - # TODO: figure out if this appropriately updates the cache - # TODO: more research on this - - name: Compare benchmarks - uses: benchmark-action/github-action-benchmark@v1 - with: - # What benchmark tool the output.txt came from - tool: 'pytest' - # Where the output from the benchmark tool is stored - output-file-path: output.json - # Where the previous data file is stored - external-data-json-path: ./cache/benchmark-data.json - # should fail consistently - alert-threshold: "120%" - # Workflow will fail when an alert happens - fail-on-alert: true - # Comment on the PR if the branch is not a fork - comment-on-alert: true - # Enable Job Summary for PRs - summary-always: true - # Always leave a comment - comment-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} From b04e14f310c456e4f9cdd38cbcac8960f24ff1ab Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 25 Nov 2024 06:54:58 -0500 Subject: [PATCH 49/82] fixed syntax error --- .github/workflows/compare_post_benchmark.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index 69f63c10..755635ed 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -5,14 +5,14 @@ #TODO: SET UP TO RUN AFTER THE BENCHMARKING IS DONE (FOLLOW YEW GUIDE!) name: "Post benchmark results" +on: [push, pull_request] -on: +#on: # replace this after other stuff starts functioning # workflow_run: # workflows: ["Tests"] # types: # - completed -on: [push, pull_request] jobs: # TODO: implement benchmarking that doesn't rely on caching -- probably via pushing to another git repository or similar @@ -106,7 +106,7 @@ jobs: # should only download from current workflow - name: Download result artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: github-token: "${{ secrets.GITHUB_TOKEN }}" pattern: ${{ github.run_id }}-${{ github.run_attempt }}-* From 95fcfce0f07af5fa6c521f73ead27f44aa0b379b Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 25 Nov 2024 06:56:36 -0500 Subject: [PATCH 50/82] commented out line to be re-added later --- .github/workflows/compare_post_benchmark.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index 755635ed..88c7dd76 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -28,7 +28,8 @@ jobs: # note: need to separate the push attempt and do so only if we are pushing to "main" ie (I THINK) autopush: false, all else should be comfortable # if this works then gh-pages only updated with a push when it is to "main", and all else will just compare hopefully, I THINK I CAN GET AWAY WITH DOING "autopush: (fix syntax here) branch == "main"" benchmark: - if: github.event.workflow_run.conclusion == 'success' + # uncomment when stuff is working + #if: github.event.workflow_run.conclusion == 'success' name: Performance Benchmark env: ENV_NAME: pyuvsim_tests_mpich From 2058a4cd77cb799831611de2a4fa26f591718062 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 25 Nov 2024 07:00:27 -0500 Subject: [PATCH 51/82] failed to comment out another line that skipped second part of workflow --- .github/workflows/compare_post_benchmark.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index 88c7dd76..ac1d178b 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -98,7 +98,7 @@ jobs: collate-post-benchmark: name: Concatenate and Post Benchmark Results needs: benchmark - if: github.event.workflow_run.conclusion == 'success' + #if: github.event.workflow_run.conclusion == 'success' runs-on: ubuntu-latest steps: From cc511cb36583e197fef356ae23a43166f648acce Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 25 Nov 2024 08:53:17 -0500 Subject: [PATCH 52/82] test for python script which concatenates benchmarks --- .github/workflows/compare_post_benchmark.yaml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index ac1d178b..297679c5 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -105,6 +105,10 @@ jobs: # Checkout repo for the github-action-benchmark action - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3.x' + # should only download from current workflow - name: Download result artifacts uses: actions/download-artifact@v4 @@ -116,6 +120,32 @@ jobs: - name: Display structure of downloaded files run: ls -R + # test approach to putting all the benchmark output in one file + - uses: jannekem/run-python-script-action@v1 + with: + script: | + import os + import json + + test_arr = [os.path.join('artifacts', bench) for bench in os.listdir('artifacts') if not bench.startswith('.')] + print(test_arr) + + output_jsons = [] + + for file in test_arr: + with open(file) as f: + output_jsons.append(json.load(f)) + + net_json = output_jsons[0] + + for ind in range(len(output_jsons)-1): + net_json['benchmarks'].append(output_jsons[ind+1]['benchmarks'][0]) + + with open('output.json', 'w') as f: + json.dump(net_json, f) + + - name: Display structure of downloaded files + run: ls -R # TODO: swap to be pytest and the works From 068699be6e907019de1e20e63a17e5739f9f9616 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 25 Nov 2024 09:22:14 -0500 Subject: [PATCH 53/82] intermediate commit --- .github/workflows/compare_post_benchmark.yaml | 26 +++++++++++++++++++ tests/test_run_ref.py | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index 297679c5..a0b062a1 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -148,6 +148,32 @@ jobs: run: ls -R + # TODO: figure out if this appropriately updates the cache + # this step also EDITS the ./cache/benchmark-data.json file + # We do not need to add output.json to the cache directory + # 2 + - name: Compare benchmarks + uses: benchmark-action/github-action-benchmark@v1 + with: + # What benchmark tool the output.txt came from + tool: 'pytest' + # Where the output from the benchmark tool is stored + output-file-path: output.json + # Where the previous data file is stored + external-data-json-path: ./cache/benchmark-data.json + # should fail consistently + alert-threshold: "120%" + # Workflow will fail when an alert happens + fail-on-alert: true + # Comment on the PR if the branch is not a fork + comment-on-alert: true + # Enable Job Summary for PRs + summary-always: true + # Always leave a comment + comment-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + + # TODO: swap to be pytest and the works #- name: Store benchmark result # uses: benchmark-action/github-action-benchmark@v1 diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index ae4e77d0..0b254379 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -141,7 +141,7 @@ def goto_tempdir(tmpdir): # list of sims to benchmark pedantically as they are too long to run many times -# TODO: if need be swap to a dictionary with more specific custom pedantic arguments +# if need be can swap to a dictionary with more specific custom pedantic arguments long_ref_sims = ["1.2_uniform", "1.2_gauss"] # TODO: generic comment on top about how this is used (conftest command line / workflow setup) From c1ca768546f38b66ad3a14921c33a41d1a388027 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 25 Nov 2024 09:36:15 -0500 Subject: [PATCH 54/82] first attempt at gh-pages approach --- .github/workflows/compare_post_benchmark.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index a0b062a1..ff7d1865 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -160,7 +160,6 @@ jobs: # Where the output from the benchmark tool is stored output-file-path: output.json # Where the previous data file is stored - external-data-json-path: ./cache/benchmark-data.json # should fail consistently alert-threshold: "120%" # Workflow will fail when an alert happens @@ -172,6 +171,8 @@ jobs: # Always leave a comment comment-always: true github-token: ${{ secrets.GITHUB_TOKEN }} + # Push and deploy GitHub pages branch automatically + auto-push: true # TODO: swap to be pytest and the works From 9f36bdd711f817a0c3a789382166231edc919ad4 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 25 Nov 2024 09:46:52 -0500 Subject: [PATCH 55/82] dummy change --- .github/workflows/compare_post_benchmark.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index ff7d1865..0ec4ca0e 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -175,6 +175,8 @@ jobs: auto-push: true + +# 1 # TODO: swap to be pytest and the works #- name: Store benchmark result # uses: benchmark-action/github-action-benchmark@v1 From 09ebd4385f550e9090ff0eac916c99ecea9e95d4 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 27 Nov 2024 16:03:15 -0500 Subject: [PATCH 56/82] preliminary approach to only pushing results of benchmark if push to 'main', and running only if Tests finishes successfully --- .github/workflows/compare_post_benchmark.yaml | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index 0ec4ca0e..4e1a713f 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -5,14 +5,18 @@ #TODO: SET UP TO RUN AFTER THE BENCHMARKING IS DONE (FOLLOW YEW GUIDE!) name: "Post benchmark results" -on: [push, pull_request] - -#on: +# on: [push, pull_request] +on: # replace this after other stuff starts functioning -# workflow_run: -# workflows: ["Tests"] -# types: -# - completed + workflow_run: + workflows: ["Tests"] + types: + - completed + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: # TODO: implement benchmarking that doesn't rely on caching -- probably via pushing to another git repository or similar @@ -29,7 +33,7 @@ jobs: # if this works then gh-pages only updated with a push when it is to "main", and all else will just compare hopefully, I THINK I CAN GET AWAY WITH DOING "autopush: (fix syntax here) branch == "main"" benchmark: # uncomment when stuff is working - #if: github.event.workflow_run.conclusion == 'success' + if: github.event.workflow_run.conclusion == 'success' name: Performance Benchmark env: ENV_NAME: pyuvsim_tests_mpich @@ -144,9 +148,11 @@ jobs: with open('output.json', 'w') as f: json.dump(net_json, f) - - name: Display structure of downloaded files - run: ls -R - + - name: Print things + run: | + echo ${{ github.event_name }} + echo ${{ github.ref_name }} + echo ${{ github.event_name == 'push' && github.ref_name == '1.1_ref_sim_ci_workflow' }} # TODO: figure out if this appropriately updates the cache # this step also EDITS the ./cache/benchmark-data.json file @@ -172,8 +178,8 @@ jobs: comment-always: true github-token: ${{ secrets.GITHUB_TOKEN }} # Push and deploy GitHub pages branch automatically - auto-push: true - + auto-push: ${{ github.event_name == 'push' && github.ref_name == '1.1_ref_sim_ci_workflow' }} + save-data-file: ${{ github.event_name == 'push' && github.ref_name == '1.1_ref_sim_ci_workflow' }} # 1 From 994a7f13dcef41aeeb8aa1afdb8d8f49fde76179 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 27 Nov 2024 16:15:58 -0500 Subject: [PATCH 57/82] removed dependence on Tests as that workflow seems to be failing independently --- .github/workflows/compare_post_benchmark.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index 4e1a713f..c0689dcf 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -5,13 +5,13 @@ #TODO: SET UP TO RUN AFTER THE BENCHMARKING IS DONE (FOLLOW YEW GUIDE!) name: "Post benchmark results" -# on: [push, pull_request] -on: -# replace this after other stuff starts functioning - workflow_run: - workflows: ["Tests"] - types: - - completed + on: [push, pull_request] +#on: +## replace this after other stuff starts functioning +# workflow_run: +# workflows: ["Tests"] +# types: +# - completed concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} From dd836e961e4458c09a677a53bc96811713a9da27 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 27 Nov 2024 16:19:52 -0500 Subject: [PATCH 58/82] hopefully fixed yaml syntax --- .github/workflows/compare_post_benchmark.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index c0689dcf..5d2cf75d 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -1,11 +1,5 @@ -#TODO: configure run on benchmark action completion! -#TODO: setup -#TODO: load artifacts -#TODO: collate artifacts (for now skip!) -#TODO: SET UP TO RUN AFTER THE BENCHMARKING IS DONE (FOLLOW YEW GUIDE!) - name: "Post benchmark results" - on: [push, pull_request] +on: [push, pull_request] #on: ## replace this after other stuff starts functioning # workflow_run: @@ -33,7 +27,7 @@ jobs: # if this works then gh-pages only updated with a push when it is to "main", and all else will just compare hopefully, I THINK I CAN GET AWAY WITH DOING "autopush: (fix syntax here) branch == "main"" benchmark: # uncomment when stuff is working - if: github.event.workflow_run.conclusion == 'success' + #if: github.event.workflow_run.conclusion == 'success' name: Performance Benchmark env: ENV_NAME: pyuvsim_tests_mpich From 43bbf9ce6c9fcc927c5b6cdd1123920a53b9a3f2 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 2 Dec 2024 13:40:21 -0500 Subject: [PATCH 59/82] added initial output statistics to the reference simulation comparisons. currently only asserts '==' can implement others or even an absolute check --- .github/workflows/compare_post_benchmark.yaml | 94 ++++++++++--------- tests/test_run_ref.py | 51 ++++++++-- 2 files changed, 96 insertions(+), 49 deletions(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index 5d2cf75d..f42575d3 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -13,20 +13,16 @@ concurrency: jobs: - # TODO: implement benchmarking that doesn't rely on caching -- probably via pushing to another git repository or similar - # TODO: can technically improve this by grouping reference sims by generation in the matrix and then have them be parametrized in batches! (DO THIS!) (not sure how to nicely do this) - # TODO: swap to saving artifact for each sim run with slightly custom json (?) follow yew - # TODO: write script that concatenates the json output of multiple benchmark runs! (get this working first) - # make artifacts path for safety reasons - # configure download of artifacts according to the actions and have them be in the one path with merge - # specify the downloading "name" using pattern (for now just *json or something) - # (obviously add this to the other yaml file for post benchmark runs, then once we have the post-benchmark run outputing the found artifacts and this benchmark successfully saving the artifacts (and clarified the uniqueness of artifacts across commits! because idk how that is kept separate and if it isn't separate need to delete / avoid that issue!)) - # then can go back to concatenation (or figure out how best to simplify that / scriptwise!) - # then can work on the github-action-benchmark on gh-pages and see how that looks once it's all concatenated, and determine if we need the write permissions - # note: need to separate the push attempt and do so only if we are pushing to "main" ie (I THINK) autopush: false, all else should be comfortable - # if this works then gh-pages only updated with a push when it is to "main", and all else will just compare hopefully, I THINK I CAN GET AWAY WITH DOING "autopush: (fix syntax here) branch == "main"" benchmark: - # uncomment when stuff is working + # Job that clones pyuvsim@main, sets up a conda environment with the necessary dependencies, + # then locally installs pyuvsim and additionally installs pytest-benchmark and requests from + # PYPI. Runs in parallel as a matrix with input individual reference simulations. The "id" + # input is passed as a flag to pytest which parametrizes the reference simulation test + # function. pytest-benchmark output is saved as an artifact with its current workflow run + # and attempt as part of name key. + + # TODO: can technically improve this by grouping reference sims by generation in the matrix and then have them be parametrized in batches! (DO THIS!) (not sure how to nicely do this) + # uncomment when Tests is working and done testing #if: github.event.workflow_run.conclusion == 'success' name: Performance Benchmark env: @@ -71,19 +67,23 @@ jobs: exit 1; fi - # also install benchmark utility and requests + # pip install benchmark utility and requests from PYPI, and local install pyuvsim - name: Install run: | pip install pytest-benchmark pip install requests pip install . + # make the artifacts directory, then run pytest using mpiexec with only 1 node and core, specifying the + # reference simulation to run using the "refsim" flag. Save the pytest benchmark output in artifacts/ + # with a sufficiently unique name - name: Run benchmark run: | - # TODO: determine if can specify a directory for file output if directory exists (if so delete third line in ez modification) mkdir artifacts/ mpiexec -n 1 -np 1 pytest --refsim=${{ matrix.id }} --benchmark-only --benchmark-json artifacts/output_${{ matrix.id }}.json -s + # upload the benchmark output as an artifact with name key corresponding to the current + # workflow run and attempt only store artifacts for 1 day - name: Upload result artifacts uses: actions/upload-artifact@v4 with: @@ -94,6 +94,14 @@ jobs: retention-days: 1 collate-post-benchmark: + # Job that loads the saved artifacts corresponding to the specific workflow run and attempt id, + # then creates a net benchmark output file named output.json with a python script action. The + # net benchmark file should still be accurate except the explicit machine info will be mostly + # lost. The net benchmark file is then fed to github-actions-benchmark which compares the + # current benchmark output with the latest data in the gh-pages branch. If the current workflow + # is a push to main, github-actions-benchmark then pushes the current benchmark output to + # gh-pages. If, during the benchmark comparison, a performance regression occurs, a comment of + # the benchmark comparison output is made on the workflow and this Job fails. name: Concatenate and Post Benchmark Results needs: benchmark #if: github.event.workflow_run.conclusion == 'success' @@ -103,11 +111,13 @@ jobs: # Checkout repo for the github-action-benchmark action - uses: actions/checkout@v4 + # setup python - uses: actions/setup-python@v4 with: python-version: '3.x' - # should only download from current workflow + # only downloads artifacts from current workflow and run attempt via the pattern matching + # loads the saved benchmark artifacts from running the benchmark matrix into artifacts/ - name: Download result artifacts uses: actions/download-artifact@v4 with: @@ -115,43 +125,58 @@ jobs: pattern: ${{ github.run_id }}-${{ github.run_attempt }}-* merge-multiple: true path: artifacts + + # prints directory info recursively, removable + # (could maybe swap this to exa or lsd because base ls doesn't do tree) - name: Display structure of downloaded files run: ls -R - # test approach to putting all the benchmark output in one file + # approach to putting all the benchmark output in one file, with the machine/run info + # of only one the pytest benchmark runs. Loads all the benchmark output artifact files, + # then takes the benchmark timing infor fron n-1 files and adds it to the first file. + # With this approach, benchmark comparison output is only a single table, and we only + # have one comment on alert in the workflow. - uses: jannekem/run-python-script-action@v1 with: script: | import os import json - test_arr = [os.path.join('artifacts', bench) for bench in os.listdir('artifacts') if not bench.startswith('.')] - print(test_arr) + # make list of paths to artifact files excluding hidden files + filepath_arr = [os.path.join('artifacts', bench) for bench in os.listdir('artifacts') if not bench.startswith('.')] + print(filepath_arr) output_jsons = [] - for file in test_arr: - with open(file) as f: + # open each filepath in the filepath_arr, load it as a json, and append it to an empty list + for filepath in filepath_arr: + with open(filepath) as f: output_jsons.append(json.load(f)) + # choose the first json as the one to modify to contain all the benchmark data net_json = output_jsons[0] - for ind in range(len(output_jsons)-1): - net_json['benchmarks'].append(output_jsons[ind+1]['benchmarks'][0]) + # iterate through the other jsons (1-n) and append their benchmark data to net_json + for json in output_jsons[1:]: + net_json['benchmarks'].append(json['benchmarks'][0]) + # save net_json as json with name output.json in current working directory with open('output.json', 'w') as f: json.dump(net_json, f) + # TODO: FIXME: decide what is novel output for the run (maybe just make it look better?) - name: Print things run: | echo ${{ github.event_name }} echo ${{ github.ref_name }} echo ${{ github.event_name == 'push' && github.ref_name == '1.1_ref_sim_ci_workflow' }} - # TODO: figure out if this appropriately updates the cache - # this step also EDITS the ./cache/benchmark-data.json file - # We do not need to add output.json to the cache directory - # 2 + # Compares the data from the specified "output-file-path" and compares + # with the latest data from the gh-pages branch. If performance regression + # occurs, fails the test and alerts. If no performance regression, (TODO: FIXME: Currently still comments, but will only comment in final version if performance regression has occurred) + # NOTE: it is important that this does not modify gh-pages on pull request + # https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#caveats + # This only updates gh-pages if a push to main occurs - name: Compare benchmarks uses: benchmark-action/github-action-benchmark@v1 with: @@ -174,18 +199,3 @@ jobs: # Push and deploy GitHub pages branch automatically auto-push: ${{ github.event_name == 'push' && github.ref_name == '1.1_ref_sim_ci_workflow' }} save-data-file: ${{ github.event_name == 'push' && github.ref_name == '1.1_ref_sim_ci_workflow' }} - - -# 1 -# TODO: swap to be pytest and the works -#- name: Store benchmark result -# uses: benchmark-action/github-action-benchmark@v1 -# with: -# name: My Project Go Benchmark -# tool: 'go' -# output-file-path: output.txt -# # Set auto-push to false since GitHub API token is not given -# auto-push: false # todo put in logic if branch == main && push or something! -# Push gh-pages branch by yourself -#- name: Push benchmark result -# run: git push 'https://you:${{ secrets.GITHUB_TOKEN }}@github.com/you/repo-name.git' gh-pages:gh-pages diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 0b254379..0bd57ad6 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -102,18 +102,55 @@ def download_sim(target_dir, sim_name): urllib.request.urlretrieve(download_url, fname) + # TODO: make more complete def compare_uvh5(uv_ref, uv_new): - # fix part(s) that should deviate - # TODO: maybe assert that deviation occurred - uv_new.history = uv_ref.history - - # perform equality check between historical and current reference + import numpy as np + + # print histories + print("History 1:") + print(uv_ref.history) + print("") + + print("History 2:") + print(uv_new.history) + print("") + + ref_arr, new_arr = uv_ref.data_array, uv_new.data_array + + # mean diff + mean_diff_of_vis = np.mean(np.abs(new_arr - ref_arr)) + mean_diff_of_abs = np.mean(np.abs(ref_arr) - np.abs(new_arr)) + + # compute max absolute difference and max relative difference + # should be identical to the allclose output for maximum absolute + # and relative difference + max_absolute_diff=np.amax(np.abs(new_arr - ref_arr)) + frac_diff=frac_diff=np.abs(new_arr - ref_arr)/np.abs(ref_arr) + frac_diff[np.isnan(frac_diff)]=0 + max_relative_diff=np.amax(frac_diff) + + # generate a true/false ndarray for passing and failing cases + # should match output of np.testing.assert_allclose + cases = np.abs(new_arr - ref_arr) <= (1e-8 + 1e-5 * np.abs(ref_arr)) + outcome = cases.all() + num_mismatched = str(len(cases[cases == False])) + "/" + str(cases.size) + + # print some things for reference + print(f"mean of abs of diff of visibilities 'mean(abs(old_data_arr - new_data_arr)): {mean_diff_of_vis}") + print(f"mean of diff of abs of visibilities 'mean(abs(old_data_arr) - abs(new_data_arr)): {mean_diff_of_abs}") + print(f"max_absolute_diff: {max_absolute_diff}") + print(f"max_relative_diff: {max_relative_diff}") + print(f"outcome: {outcome}") + print(f"num_mismatched: {num_mismatched}") + + # perform basic equality check between historical and current reference # simulation output - # TODO: implement better asserts - # include lower tolerance for deviations assert uv_new == uv_ref + # perform more intensive check + # TODO + def construct_filepaths(target_dir, sim): # construct filename and filepath of downloaded file uvh5_filename = "ref_" + sim + ".uvh5" From 5766fadb3895c4a2ebc929f0519067c429477488 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 2 Dec 2024 13:44:58 -0500 Subject: [PATCH 60/82] re-added setting history to be equal --- tests/test_run_ref.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 0bd57ad6..f7f112c1 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -116,6 +116,9 @@ def compare_uvh5(uv_ref, uv_new): print(uv_new.history) print("") + # set history to match so equality check doesn't fail + uv_new.history = uv_ref.history + ref_arr, new_arr = uv_ref.data_array, uv_new.data_array # mean diff From 62779e46418b7be104446d5a2e5338fef1e409a5 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 2 Dec 2024 13:50:00 -0500 Subject: [PATCH 61/82] fix --- .github/workflows/compare_post_benchmark.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index f42575d3..7cf2a35f 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -157,8 +157,8 @@ jobs: net_json = output_jsons[0] # iterate through the other jsons (1-n) and append their benchmark data to net_json - for json in output_jsons[1:]: - net_json['benchmarks'].append(json['benchmarks'][0]) + for json_out in output_jsons[1:]: + net_json['benchmarks'].append(json_out['benchmarks'][0]) # save net_json as json with name output.json in current working directory with open('output.json', 'w') as f: From d20f4158cae2967a7a0eb453aa304753bdbf2b5e Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 2 Dec 2024 19:00:50 -0500 Subject: [PATCH 62/82] all current ref sims should run now, and implemented hopefully more robust downloading --- .github/workflows/compare_post_benchmark.yaml | 12 ++-- tests/test_run_ref.py | 55 +++++++++++-------- 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index 7cf2a35f..acab009e 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -31,16 +31,16 @@ jobs: runs-on: ubuntu-latest strategy: - #max-parallel: 1 + # all jobs should run in parallel (unless implementing above TODO) matrix: include: - id: 1.1_uniform - id: 1.1_gauss -# - id: 1.1_mwa -# - id: 1.2_uniform -# - id: 1.2_gauss -# - id: 1.3_uniform -# - id: 1.3_gauss + - id: 1.1_mwa + - id: 1.2_uniform + - id: 1.2_gauss + - id: 1.3_uniform + - id: 1.3_gauss defaults: run: diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index f7f112c1..943f6697 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -16,12 +16,21 @@ pytest.importorskip("mpi4py") # noqa -#TODO: make downloading simulation output file and companion files more robust (add retries, etc...) -# gets latest pid from api call -def get_latest_pid(response): +# attempt to GET the url n_retry times (if return code in range) +def robust_response(url, n_retry=5): import requests + from requests.adapters import HTTPAdapter, Retry + + s = requests.Session() + retries = Retry(total=n_retry, backoff_factor=1, raise_on_status=True, status_forcelist=range(500, 600)) + s.mount('http://', HTTPAdapter(max_retries=retries)) + + return s.get(url) + +# gets latest pid from api call +def get_latest_pid(response): # list of item dicts collection_response_items = response.json()['items']['docs'] @@ -36,7 +45,7 @@ def get_latest_pid(response): pids = [] for uri in json_uris: # get response for item as json - response_json = requests.get(uri).json() + response_json = robust_response(uri).json() # get values from json time_created = response_json['object_created_dsi'] @@ -60,13 +69,11 @@ def get_latest_pid(response): # TODO: fix up order of operations and comment the method better def download_sim(target_dir, sim_name): - import requests - import urllib.request - + # api url api_url="https://repository.library.brown.edu/api/collections/bdr:wte2qah8/?q=" print(f"\nquerying BDR collection for items matching {sim_name} via api: {api_url+sim_name}") - response=requests.get(api_url+sim_name) + response=robust_response(api_url+sim_name) # parse out the latest file in the collection from the search result and return its pid pid = get_latest_pid(response) @@ -85,22 +92,25 @@ def download_sim(target_dir, sim_name): # download url download_url = f"https://repository.library.brown.edu/storage/{pid}/content/" - print(f"attempting download of requested file by http: {download_url}\n") # download the file to the location - urllib.request.urlretrieve(download_url, fname) + print(f"attempting download of requested file by http: {download_url}\n") + bdr_file_response = robust_response(download_url) + with open(fname, 'wb') as f: + f.write(bdr_file_response.content) # additionally download mwa uvbeam to SIM_DATA_PATH if necessary if "mwa" in sim_name: download_url = "http://ws.mwatelescope.org/static/mwa_full_embedded_element_pattern.h5" fname=os.path.join(SIM_DATA_PATH,"mwa_full_embedded_element_pattern.h5") - print(fname) - # download the file to the location + # download the file to the location if not already downloaded if os.path.isfile(fname): print("skipping download mwa uvbeam file already downloaded") else: - urllib.request.urlretrieve(download_url, fname) - + print(f"attempting download of requested file by http: {download_url}\n") + mwa_beam_response = robust_response(download_url) + with open(fname, 'wb') as f: + f.write(mwa_beam_response.content) # TODO: make more complete @@ -147,12 +157,13 @@ def compare_uvh5(uv_ref, uv_new): print(f"outcome: {outcome}") print(f"num_mismatched: {num_mismatched}") - # perform basic equality check between historical and current reference - # simulation output + # perform equality check between historical and current reference + # simulation output assert uv_new == uv_ref + + # fail if any element of data_array is different between new and ref + assert (ref_arr == new_arr).all() - # perform more intensive check - # TODO def construct_filepaths(target_dir, sim): # construct filename and filepath of downloaded file @@ -180,10 +191,6 @@ def goto_tempdir(tmpdir): os.chdir(cwd) -# list of sims to benchmark pedantically as they are too long to run many times -# if need be can swap to a dictionary with more specific custom pedantic arguments -long_ref_sims = ["1.2_uniform", "1.2_gauss"] - # TODO: generic comment on top about how this is used (conftest command line / workflow setup) @pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") def test_run_sim(benchmark, goto_tempdir, refsim): @@ -197,6 +204,10 @@ def test_run_sim(benchmark, goto_tempdir, refsim): # file as the point of historical comparison uv_ref = UVData.from_file(uvh5_filepath) + # list of sims to benchmark pedantically as they are too long to run many times + # if need be can swap to a dictionary with more specific custom pedantic arguments + long_ref_sims = ["1.2_uniform", "1.2_gauss"] + # benchmark uvsim.run_uvsim method with param_filename argument # runs multiple times to get benchmark data # outdir is given by the yaml file but should not write anything From 57320fecfcd64c9e2a73dc1a9c6f3427111e5629 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 2 Dec 2024 19:13:50 -0500 Subject: [PATCH 63/82] commented out the 0 tolerance sim comparison check --- tests/test_run_ref.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 943f6697..26c78f70 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -162,7 +162,9 @@ def compare_uvh5(uv_ref, uv_new): assert uv_new == uv_ref # fail if any element of data_array is different between new and ref - assert (ref_arr == new_arr).all() + # currently commented out but might be worth implementing and updating ref_sim file each time + # the assertion fails for the relevant files + # assert (ref_arr == new_arr).all() def construct_filepaths(target_dir, sim): From 0a4f02e0c3fa3801209968fccddd4a2ececd4101 Mon Sep 17 00:00:00 2001 From: burdorfmitchell <90595017+burdorfmitchell@users.noreply.github.com> Date: Mon, 2 Dec 2024 20:10:39 -0500 Subject: [PATCH 64/82] added dummy counter (#513) * added dummy counter * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/compare_post_benchmark.yaml | 19 ++-- tests/conftest.py | 6 +- tests/test_run_ref.py | 93 +++++++++++-------- 3 files changed, 70 insertions(+), 48 deletions(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index acab009e..69205038 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -20,7 +20,7 @@ jobs: # input is passed as a flag to pytest which parametrizes the reference simulation test # function. pytest-benchmark output is saved as an artifact with its current workflow run # and attempt as part of name key. - + # TODO: can technically improve this by grouping reference sims by generation in the matrix and then have them be parametrized in batches! (DO THIS!) (not sure how to nicely do this) # uncomment when Tests is working and done testing #if: github.event.workflow_run.conclusion == 'success' @@ -48,7 +48,7 @@ jobs: shell: bash -l {0} steps: - uses: actions/checkout@main - + - name: Setup Miniforge uses: conda-incubator/setup-miniconda@v3 with: @@ -74,15 +74,15 @@ jobs: pip install requests pip install . - # make the artifacts directory, then run pytest using mpiexec with only 1 node and core, specifying the + # make the artifacts directory, then run pytest using mpiexec with only 1 node and core, specifying the # reference simulation to run using the "refsim" flag. Save the pytest benchmark output in artifacts/ # with a sufficiently unique name - name: Run benchmark run: | mkdir artifacts/ - mpiexec -n 1 -np 1 pytest --refsim=${{ matrix.id }} --benchmark-only --benchmark-json artifacts/output_${{ matrix.id }}.json -s + mpiexec -n 1 -np 1 pytest --refsim=${{ matrix.id }} --benchmark-only --benchmark-json artifacts/output_${{ matrix.id }}.json -s - # upload the benchmark output as an artifact with name key corresponding to the current + # upload the benchmark output as an artifact with name key corresponding to the current # workflow run and attempt only store artifacts for 1 day - name: Upload result artifacts uses: actions/upload-artifact@v4 @@ -94,12 +94,12 @@ jobs: retention-days: 1 collate-post-benchmark: - # Job that loads the saved artifacts corresponding to the specific workflow run and attempt id, + # Job that loads the saved artifacts corresponding to the specific workflow run and attempt id, # then creates a net benchmark output file named output.json with a python script action. The # net benchmark file should still be accurate except the explicit machine info will be mostly - # lost. The net benchmark file is then fed to github-actions-benchmark which compares the + # lost. The net benchmark file is then fed to github-actions-benchmark which compares the # current benchmark output with the latest data in the gh-pages branch. If the current workflow - # is a push to main, github-actions-benchmark then pushes the current benchmark output to + # is a push to main, github-actions-benchmark then pushes the current benchmark output to # gh-pages. If, during the benchmark comparison, a performance regression occurs, a comment of # the benchmark comparison output is made on the workflow and this Job fails. name: Concatenate and Post Benchmark Results @@ -171,12 +171,13 @@ jobs: echo ${{ github.ref_name }} echo ${{ github.event_name == 'push' && github.ref_name == '1.1_ref_sim_ci_workflow' }} + # counter for dummy commmit: 1 # Compares the data from the specified "output-file-path" and compares # with the latest data from the gh-pages branch. If performance regression # occurs, fails the test and alerts. If no performance regression, (TODO: FIXME: Currently still comments, but will only comment in final version if performance regression has occurred) # NOTE: it is important that this does not modify gh-pages on pull request # https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#caveats - # This only updates gh-pages if a push to main occurs + # This only updates gh-pages if a push to main occurs - name: Compare benchmarks uses: benchmark-action/github-action-benchmark@v1 with: diff --git a/tests/conftest.py b/tests/conftest.py index d36dca0b..0eff7956 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -63,8 +63,10 @@ def pytest_addoption(parser): "--nompi", action="store_true", help="skip mpi-parallelized tests." ) parser.addoption( - "--refsim", action="append", default=[], - help="list of refsim names to pass to test functions." + "--refsim", + action="append", + default=[], + help="list of refsim names to pass to test functions.", ) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 26c78f70..2869a00e 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -8,7 +8,8 @@ from pyuvdata import UVData from pyuvsim.data import DATA_PATH as SIM_DATA_PATH -#from pyuvsim.mpi import rank + +# from pyuvsim.mpi import rank import pyuvsim from pyuvsim.uvsim import run_uvsim @@ -23,24 +24,31 @@ def robust_response(url, n_retry=5): from requests.adapters import HTTPAdapter, Retry s = requests.Session() - retries = Retry(total=n_retry, backoff_factor=1, raise_on_status=True, status_forcelist=range(500, 600)) - s.mount('http://', HTTPAdapter(max_retries=retries)) + retries = Retry( + total=n_retry, + backoff_factor=1, + raise_on_status=True, + status_forcelist=range(500, 600), + ) + s.mount("http://", HTTPAdapter(max_retries=retries)) - return s.get(url) + return s.get(url) # gets latest pid from api call def get_latest_pid(response): # list of item dicts - collection_response_items = response.json()['items']['docs'] + collection_response_items = response.json()["items"]["docs"] if len(collection_response_items) == 0: return "No items found in collection matching query." # using "object_created_dsi" key to sort the items, so need to request that # for each item via the "json_uri", and get the pid as well to return - print(f"requesting json_uri for each item in reponse, and parsing 'object_created_dsi' and 'pid'") - json_uris = [ item['json_uri'] for item in collection_response_items ] + print( + f"requesting json_uri for each item in reponse, and parsing 'object_created_dsi' and 'pid'" + ) + json_uris = [item["json_uri"] for item in collection_response_items] object_created_dsis = [] pids = [] for uri in json_uris: @@ -48,8 +56,8 @@ def get_latest_pid(response): response_json = robust_response(uri).json() # get values from json - time_created = response_json['object_created_dsi'] - item_pid = response_json['pid'] + time_created = response_json["object_created_dsi"] + item_pid = response_json["pid"] # append to lists object_created_dsis.append(time_created) @@ -58,22 +66,26 @@ def get_latest_pid(response): # get the max timestamp, then get the index at which the max time occurs # this corresponds to the latest created object latest_item_pos = object_created_dsis.index(max(object_created_dsis)) - + # get the pid of the latest item by shared pos latest_pid = pids[latest_item_pos] - print("returning pid of most recent file uploaded to the collection matching the query") - + print( + "returning pid of most recent file uploaded to the collection matching the query" + ) + return latest_pid # TODO: fix up order of operations and comment the method better def download_sim(target_dir, sim_name): # api url - api_url="https://repository.library.brown.edu/api/collections/bdr:wte2qah8/?q=" - - print(f"\nquerying BDR collection for items matching {sim_name} via api: {api_url+sim_name}") - response=robust_response(api_url+sim_name) + api_url = "https://repository.library.brown.edu/api/collections/bdr:wte2qah8/?q=" + + print( + f"\nquerying BDR collection for items matching {sim_name} via api: {api_url+sim_name}" + ) + response = robust_response(api_url + sim_name) # parse out the latest file in the collection from the search result and return its pid pid = get_latest_pid(response) @@ -95,13 +107,15 @@ def download_sim(target_dir, sim_name): # download the file to the location print(f"attempting download of requested file by http: {download_url}\n") bdr_file_response = robust_response(download_url) - with open(fname, 'wb') as f: + with open(fname, "wb") as f: f.write(bdr_file_response.content) # additionally download mwa uvbeam to SIM_DATA_PATH if necessary if "mwa" in sim_name: - download_url = "http://ws.mwatelescope.org/static/mwa_full_embedded_element_pattern.h5" - fname=os.path.join(SIM_DATA_PATH,"mwa_full_embedded_element_pattern.h5") + download_url = ( + "http://ws.mwatelescope.org/static/mwa_full_embedded_element_pattern.h5" + ) + fname = os.path.join(SIM_DATA_PATH, "mwa_full_embedded_element_pattern.h5") # download the file to the location if not already downloaded if os.path.isfile(fname): @@ -109,7 +123,7 @@ def download_sim(target_dir, sim_name): else: print(f"attempting download of requested file by http: {download_url}\n") mwa_beam_response = robust_response(download_url) - with open(fname, 'wb') as f: + with open(fname, "wb") as f: f.write(mwa_beam_response.content) @@ -138,30 +152,34 @@ def compare_uvh5(uv_ref, uv_new): # compute max absolute difference and max relative difference # should be identical to the allclose output for maximum absolute # and relative difference - max_absolute_diff=np.amax(np.abs(new_arr - ref_arr)) - frac_diff=frac_diff=np.abs(new_arr - ref_arr)/np.abs(ref_arr) - frac_diff[np.isnan(frac_diff)]=0 - max_relative_diff=np.amax(frac_diff) + max_absolute_diff = np.amax(np.abs(new_arr - ref_arr)) + frac_diff = frac_diff = np.abs(new_arr - ref_arr) / np.abs(ref_arr) + frac_diff[np.isnan(frac_diff)] = 0 + max_relative_diff = np.amax(frac_diff) # generate a true/false ndarray for passing and failing cases - # should match output of np.testing.assert_allclose + # should match output of np.testing.assert_allclose cases = np.abs(new_arr - ref_arr) <= (1e-8 + 1e-5 * np.abs(ref_arr)) outcome = cases.all() num_mismatched = str(len(cases[cases == False])) + "/" + str(cases.size) # print some things for reference - print(f"mean of abs of diff of visibilities 'mean(abs(old_data_arr - new_data_arr)): {mean_diff_of_vis}") - print(f"mean of diff of abs of visibilities 'mean(abs(old_data_arr) - abs(new_data_arr)): {mean_diff_of_abs}") + print( + f"mean of abs of diff of visibilities 'mean(abs(old_data_arr - new_data_arr)): {mean_diff_of_vis}" + ) + print( + f"mean of diff of abs of visibilities 'mean(abs(old_data_arr) - abs(new_data_arr)): {mean_diff_of_abs}" + ) print(f"max_absolute_diff: {max_absolute_diff}") print(f"max_relative_diff: {max_relative_diff}") print(f"outcome: {outcome}") print(f"num_mismatched: {num_mismatched}") # perform equality check between historical and current reference - # simulation output + # simulation output assert uv_new == uv_ref - - # fail if any element of data_array is different between new and ref + + # fail if any element of data_array is different between new and ref # currently commented out but might be worth implementing and updating ref_sim file each time # the assertion fails for the relevant files # assert (ref_arr == new_arr).all() @@ -217,13 +235,14 @@ def test_run_sim(benchmark, goto_tempdir, refsim): # uses long_ref_sims global array to determine if pedantic benchmarking should be used if refsim in long_ref_sims: uv_new = benchmark.pedantic( - run_uvsim, - args=[yaml_filepath], - kwargs={'return_uv':True}, - setup=None, - rounds=1, - warmup_rounds=0, - iterations=1) + run_uvsim, + args=[yaml_filepath], + kwargs={"return_uv": True}, + setup=None, + rounds=1, + warmup_rounds=0, + iterations=1, + ) else: uv_new = benchmark(run_uvsim, yaml_filepath, return_uv=True) From a4326c8ae9b2993b239d9ae329a7a66eabd13f7b Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Tue, 3 Dec 2024 19:49:44 -0500 Subject: [PATCH 65/82] only one TODO left to resolve in test_run_ref (determining how strict object comparison should be) --- tests/test_run_ref.py | 44 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 2869a00e..cdc40fb7 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -9,7 +9,6 @@ from pyuvsim.data import DATA_PATH as SIM_DATA_PATH -# from pyuvsim.mpi import rank import pyuvsim from pyuvsim.uvsim import run_uvsim @@ -18,8 +17,12 @@ pytest.importorskip("mpi4py") # noqa -# attempt to GET the url n_retry times (if return code in range) def robust_response(url, n_retry=5): + # attempt to GET the url n_retry times (if return code in range) + # returns the GET request + # Stackoverflow / Docs link for approach / Retry: + # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request + # https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#module-urllib3.util.retry import requests from requests.adapters import HTTPAdapter, Retry @@ -37,13 +40,18 @@ def robust_response(url, n_retry=5): # gets latest pid from api call def get_latest_pid(response): - # list of item dicts + # takes as input the response from an api call to the Brown Digital Repository for the + # collection "pyuvsim historical reference simulations". Parses the response for the latest + # uploaded created item matching the query, then returns the PID of that item to be downloaded. + # In order to parse the response, further API calls are sent to get explicit data for each + # item. If the increased number of API calls becomes an issue then this function can be changed + # to simply determine basic datetime info from the input response with no further calls. collection_response_items = response.json()["items"]["docs"] if len(collection_response_items) == 0: return "No items found in collection matching query." - # using "object_created_dsi" key to sort the items, so need to request that + # using "object_created_dsi" key to sort the items, so we need to request that # for each item via the "json_uri", and get the pid as well to return print( f"requesting json_uri for each item in reponse, and parsing 'object_created_dsi' and 'pid'" @@ -77,8 +85,16 @@ def get_latest_pid(response): return latest_pid -# TODO: fix up order of operations and comment the method better def download_sim(target_dir, sim_name): + # method to download the historical reference simulations from the Brown Digital + # Repository. Sends an api call to the "pyuvsim historical reference simulations" collection, + # then identifies the latest uploaded object in the response. Downloads that object to the + # target directory and if the object requires the mwa beam file downloads that to the + # SIM_DATA_PATH + + # Link to BDR API DOCS: + # https://repository.library.brown.edu/studio/api-docs/ + # api url api_url = "https://repository.library.brown.edu/api/collections/bdr:wte2qah8/?q=" @@ -127,8 +143,11 @@ def download_sim(target_dir, sim_name): f.write(mwa_beam_response.content) -# TODO: make more complete def compare_uvh5(uv_ref, uv_new): + # takes as input two UVData objects, and computes relevant quantities for determining how + # similar the data are. Prints the histories before setting them equal. Currently only runs + # an equality check but (TODO: FIXME) should make a much more exhaustive check OR just turn + # back on the exact check and update the sim output when it differs (Do this tbh) import numpy as np # print histories @@ -186,6 +205,10 @@ def compare_uvh5(uv_ref, uv_new): def construct_filepaths(target_dir, sim): + # takes as input the sim name (NEEDS TO BE AN EXISTING SIM IN THE DATA DIRECTORY), then + # constructs the expected yaml_filepath to run the simulation and uvh5_filepath to locate + # the downloaded historical output + # construct filename and filepath of downloaded file uvh5_filename = "ref_" + sim + ".uvh5" uvh5_filepath = os.path.join(target_dir, "results_data", uvh5_filename) @@ -194,6 +217,8 @@ def construct_filepaths(target_dir, sim): # to run simulation yaml_filename = "obsparam_ref_" + sim + ".yaml" yaml_filepath = os.path.join(SIM_DATA_PATH, "test_config", yaml_filename) + + # if yaml_filepath does not exist then comparison should fail assert os.path.exists(yaml_filepath) return uvh5_filepath, yaml_filepath @@ -211,9 +236,13 @@ def goto_tempdir(tmpdir): os.chdir(cwd) -# TODO: generic comment on top about how this is used (conftest command line / workflow setup) @pytest.mark.skipif(not hasbench, reason="benchmark utility not installed") def test_run_sim(benchmark, goto_tempdir, refsim): + # pytest method to benchmark reference simulations. currently only called on one reference + # simulation at a time. takes as input the benchmark fixture, a fixture to generate a temporary + # directory for testing, and a fixture defined in conftest.py that is used to parametrize this + # method with a specific reference simulation name via command line input. + # download reference sim output to compare download_sim(goto_tempdir, refsim) @@ -251,4 +280,5 @@ def test_run_sim(benchmark, goto_tempdir, refsim): if pyuvsim.mpi.rank != 0: return + # performs any assertions to confirm that the reference simulation output hasn't diverged compare_uvh5(uv_ref, uv_new) From 7e8bfa29a6ab6c439b614a542595c7611b6cd12f Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Tue, 3 Dec 2024 20:10:31 -0500 Subject: [PATCH 66/82] cleaned up compare_post_benchmark.yaml a bit. now need to test running compare_post_benchmark using completion of another workflow (pull request and push) --- .github/workflows/compare_post_benchmark.yaml | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index 69205038..bd37efee 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -20,8 +20,11 @@ jobs: # input is passed as a flag to pytest which parametrizes the reference simulation test # function. pytest-benchmark output is saved as an artifact with its current workflow run # and attempt as part of name key. + + # Link to discussion of artifacts + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/storing-and-sharing-data-from-a-workflow#about-workflow-artifacts + # https://github.com/actions/upload-artifact - # TODO: can technically improve this by grouping reference sims by generation in the matrix and then have them be parametrized in batches! (DO THIS!) (not sure how to nicely do this) # uncomment when Tests is working and done testing #if: github.event.workflow_run.conclusion == 'success' name: Performance Benchmark @@ -31,7 +34,7 @@ jobs: runs-on: ubuntu-latest strategy: - # all jobs should run in parallel (unless implementing above TODO) + # all jobs should run in parallel matrix: include: - id: 1.1_uniform @@ -93,6 +96,7 @@ jobs: include-hidden-files: true retention-days: 1 + collate-post-benchmark: # Job that loads the saved artifacts corresponding to the specific workflow run and attempt id, # then creates a net benchmark output file named output.json with a python script action. The @@ -102,9 +106,15 @@ jobs: # is a push to main, github-actions-benchmark then pushes the current benchmark output to # gh-pages. If, during the benchmark comparison, a performance regression occurs, a comment of # the benchmark comparison output is made on the workflow and this Job fails. + + # Inspired by this workflow by yewstack/yew and the github-action-benchmark README: + # https://github.com/yewstack/yew/blob/master/.github/workflows/benchmark.yml + # https://github.com/yewstack/yew/blob/master/.github/workflows/post-benchmark.yml + # https://github.com/benchmark-action/github-action-benchmark + # https://github.com/actions/download-artifact + name: Concatenate and Post Benchmark Results needs: benchmark - #if: github.event.workflow_run.conclusion == 'success' runs-on: ubuntu-latest steps: @@ -164,14 +174,13 @@ jobs: with open('output.json', 'w') as f: json.dump(net_json, f) - # TODO: FIXME: decide what is novel output for the run (maybe just make it look better?) + # Print github event_name and ref_name and the boolean check for whether gh-pages should be updated - name: Print things run: | - echo ${{ github.event_name }} - echo ${{ github.ref_name }} - echo ${{ github.event_name == 'push' && github.ref_name == '1.1_ref_sim_ci_workflow' }} + echo "Event Name: ${{ github.event_name }}" + echo "Ref Name: ${{ github.ref_name }}" + echo "Update gh-pages: ${{ github.event_name == 'push' && github.ref_name == '1.1_ref_sim_ci_workflow' }}" - # counter for dummy commmit: 1 # Compares the data from the specified "output-file-path" and compares # with the latest data from the gh-pages branch. If performance regression # occurs, fails the test and alerts. If no performance regression, (TODO: FIXME: Currently still comments, but will only comment in final version if performance regression has occurred) From f304d88a6ea14bbfd4e1843d16519cf17094b297 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Tue, 3 Dec 2024 21:58:43 -0500 Subject: [PATCH 67/82] updated approach to computing num_mismatched and fixed style --- tests/test_run_ref.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index cdc40fb7..46ead372 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -7,9 +7,8 @@ import pytest from pyuvdata import UVData -from pyuvsim.data import DATA_PATH as SIM_DATA_PATH - import pyuvsim +from pyuvsim.data import DATA_PATH as SIM_DATA_PATH from pyuvsim.uvsim import run_uvsim hasbench = importlib.util.find_spec("pytest_benchmark") is not None @@ -40,12 +39,12 @@ def robust_response(url, n_retry=5): # gets latest pid from api call def get_latest_pid(response): - # takes as input the response from an api call to the Brown Digital Repository for the - # collection "pyuvsim historical reference simulations". Parses the response for the latest + # takes as input the response from an api call to the Brown Digital Repository for the + # collection "pyuvsim historical reference simulations". Parses the response for the latest # uploaded created item matching the query, then returns the PID of that item to be downloaded. - # In order to parse the response, further API calls are sent to get explicit data for each + # In order to parse the response, further API calls are sent to get explicit data for each # item. If the increased number of API calls becomes an issue then this function can be changed - # to simply determine basic datetime info from the input response with no further calls. + # to simply determine basic datetime info from the input response with no further calls. collection_response_items = response.json()["items"]["docs"] if len(collection_response_items) == 0: @@ -54,7 +53,7 @@ def get_latest_pid(response): # using "object_created_dsi" key to sort the items, so we need to request that # for each item via the "json_uri", and get the pid as well to return print( - f"requesting json_uri for each item in reponse, and parsing 'object_created_dsi' and 'pid'" + "requesting json_uri for each item in reponse, and parsing 'object_created_dsi' and 'pid'" ) json_uris = [item["json_uri"] for item in collection_response_items] object_created_dsis = [] @@ -89,7 +88,7 @@ def download_sim(target_dir, sim_name): # method to download the historical reference simulations from the Brown Digital # Repository. Sends an api call to the "pyuvsim historical reference simulations" collection, # then identifies the latest uploaded object in the response. Downloads that object to the - # target directory and if the object requires the mwa beam file downloads that to the + # target directory and if the object requires the mwa beam file downloads that to the # SIM_DATA_PATH # Link to BDR API DOCS: @@ -144,8 +143,8 @@ def download_sim(target_dir, sim_name): def compare_uvh5(uv_ref, uv_new): - # takes as input two UVData objects, and computes relevant quantities for determining how - # similar the data are. Prints the histories before setting them equal. Currently only runs + # takes as input two UVData objects, and computes relevant quantities for determining how + # similar the data are. Prints the histories before setting them equal. Currently only runs # an equality check but (TODO: FIXME) should make a much more exhaustive check OR just turn # back on the exact check and update the sim output when it differs (Do this tbh) import numpy as np @@ -180,14 +179,26 @@ def compare_uvh5(uv_ref, uv_new): # should match output of np.testing.assert_allclose cases = np.abs(new_arr - ref_arr) <= (1e-8 + 1e-5 * np.abs(ref_arr)) outcome = cases.all() - num_mismatched = str(len(cases[cases == False])) + "/" + str(cases.size) + + # get unique outcomes (true / false) and corresponding counts + # then convert to dict and get result + unique, counts = np.unique(cases, return_counts=True) + outcome_dict = dict(zip(unique, counts, strict=False)) + + # need to check that key exists + if False in outcome_dict: + num_mismatched = str(outcome_dict[False]) + "/" + str(cases.size) + else: + num_mismatched = "0" + "/" + str(cases.size) # print some things for reference print( - f"mean of abs of diff of visibilities 'mean(abs(old_data_arr - new_data_arr)): {mean_diff_of_vis}" + f"mean of abs of diff of visibilities " + f"'mean(abs(old_data_arr - new_data_arr))': {mean_diff_of_vis}" ) print( - f"mean of diff of abs of visibilities 'mean(abs(old_data_arr) - abs(new_data_arr)): {mean_diff_of_abs}" + f"mean of diff of abs of visibilities " + f"'mean(abs(old_data_arr) - abs(new_data_arr))': {mean_diff_of_abs}" ) print(f"max_absolute_diff: {max_absolute_diff}") print(f"max_relative_diff: {max_relative_diff}") @@ -205,7 +216,7 @@ def compare_uvh5(uv_ref, uv_new): def construct_filepaths(target_dir, sim): - # takes as input the sim name (NEEDS TO BE AN EXISTING SIM IN THE DATA DIRECTORY), then + # takes as input the sim name (NEEDS TO BE AN EXISTING SIM IN THE DATA DIRECTORY), then # constructs the expected yaml_filepath to run the simulation and uvh5_filepath to locate # the downloaded historical output @@ -280,5 +291,5 @@ def test_run_sim(benchmark, goto_tempdir, refsim): if pyuvsim.mpi.rank != 0: return - # performs any assertions to confirm that the reference simulation output hasn't diverged + # performs any assertions to confirm that the reference simulation output hasn't diverged compare_uvh5(uv_ref, uv_new) From 72bfb1202387d7fdb99a803af3af9a6a9d075efa Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 4 Dec 2024 00:11:17 -0500 Subject: [PATCH 68/82] swapped compare_post_benchmark to run after Tests --- .github/workflows/compare_post_benchmark.yaml | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index bd37efee..f3d9d1d5 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -1,17 +1,14 @@ name: "Post benchmark results" -on: [push, pull_request] -#on: -## replace this after other stuff starts functioning -# workflow_run: -# workflows: ["Tests"] -# types: -# - completed +on: + workflow_run: + workflows: ["Tests"] + types: + - completed concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true - jobs: benchmark: # Job that clones pyuvsim@main, sets up a conda environment with the necessary dependencies, @@ -20,11 +17,11 @@ jobs: # input is passed as a flag to pytest which parametrizes the reference simulation test # function. pytest-benchmark output is saved as an artifact with its current workflow run # and attempt as part of name key. - + # # Link to discussion of artifacts # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/storing-and-sharing-data-from-a-workflow#about-workflow-artifacts # https://github.com/actions/upload-artifact - + # # uncomment when Tests is working and done testing #if: github.event.workflow_run.conclusion == 'success' name: Performance Benchmark @@ -96,7 +93,6 @@ jobs: include-hidden-files: true retention-days: 1 - collate-post-benchmark: # Job that loads the saved artifacts corresponding to the specific workflow run and attempt id, # then creates a net benchmark output file named output.json with a python script action. The @@ -106,13 +102,13 @@ jobs: # is a push to main, github-actions-benchmark then pushes the current benchmark output to # gh-pages. If, during the benchmark comparison, a performance regression occurs, a comment of # the benchmark comparison output is made on the workflow and this Job fails. - + # # Inspired by this workflow by yewstack/yew and the github-action-benchmark README: # https://github.com/yewstack/yew/blob/master/.github/workflows/benchmark.yml # https://github.com/yewstack/yew/blob/master/.github/workflows/post-benchmark.yml # https://github.com/benchmark-action/github-action-benchmark # https://github.com/actions/download-artifact - + name: Concatenate and Post Benchmark Results needs: benchmark runs-on: ubuntu-latest From 3a09cdf73d809edc031dd6b45adf9bbe846957ae Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 4 Dec 2024 00:40:08 -0500 Subject: [PATCH 69/82] minor edits to compare_post_benchmark so hopefully it runs --- .github/workflows/compare_post_benchmark.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index f3d9d1d5..b9bcc19e 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -1,13 +1,13 @@ name: "Post benchmark results" on: workflow_run: - workflows: ["Tests"] + workflows: [Tests] types: - - completed + - in_progress -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true +#concurrency: +# group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} +# cancel-in-progress: true jobs: benchmark: From c784654f319b26f4b39f8a244f37a4a3b4f7cfb3 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 4 Dec 2024 00:58:58 -0500 Subject: [PATCH 70/82] not sure why linking to tests isn't working -- swapping back --- .github/workflows/compare_post_benchmark.yaml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index b9bcc19e..9e75dea2 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -1,13 +1,14 @@ name: "Post benchmark results" -on: - workflow_run: - workflows: [Tests] - types: - - in_progress - -#concurrency: -# group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} -# cancel-in-progress: true +on: [push, pull_request] +#on: +# workflow_run: +# workflows: [Tests] +# types: +# - in_progress + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true jobs: benchmark: From 7da96f85cc7b403561bf1ab371770f11e59f07ac Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 4 Dec 2024 02:28:02 -0500 Subject: [PATCH 71/82] edited README / environment.yaml to discuss/require requests and pytest-benchmark --- README.md | 4 ++++ environment.yml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/README.md b/README.md index 448e9797..14a1f884 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,10 @@ One other package, pytest-xdist, is not required, but can be used to speed up ru the test suite by running tests in parallel. To use it call pytest with the ```-n auto``` option. +Additionally, two packages, pytest-benchmark and requests, are required if one needs to locally +run benchmarking of the reference simulations using pytest. pytest-benchmark is not necessary +otherwise, and requests is used for minor scripts in /reference_simulations. + One way to ensure you have all the needed packages is to use the included `environment.yaml` file to create a new environment that will contain all the optional dependencies along with dependencies required for diff --git a/environment.yml b/environment.yml index 97c83da7..47bb4f50 100644 --- a/environment.yml +++ b/environment.yml @@ -15,6 +15,8 @@ dependencies: - pytest - pytest-cov>=5.0.0 - pytest-xdist + - pytest-benchmark + - requests - pyuvdata>=3.1.2 - pyyaml>=5.4.1 - scipy>=1.8 From cc4851277cb7b645011765b90543da3daa79146d Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 4 Dec 2024 21:57:27 -0500 Subject: [PATCH 72/82] edited README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 14a1f884..9b63222e 100644 --- a/README.md +++ b/README.md @@ -127,9 +127,9 @@ One other package, pytest-xdist, is not required, but can be used to speed up ru the test suite by running tests in parallel. To use it call pytest with the ```-n auto``` option. -Additionally, two packages, pytest-benchmark and requests, are required if one needs to locally -run benchmarking of the reference simulations using pytest. pytest-benchmark is not necessary -otherwise, and requests is used for minor scripts in /reference_simulations. +Two additional packages, pytest-benchmark and requests, are required if one needs to locally run +single core regression testing of the reference simulations. For more realistic benchmarking at +any level of scale, see [Benchmarking](https://pyuvsim.readthedocs.io/en/latest/developers.html#benchmarking). One way to ensure you have all the needed packages is to use the included `environment.yaml` file to create a new environment that will From b80aae5e27aca8a4392b4cd7637a508192317bcd Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 4 Dec 2024 23:52:29 -0500 Subject: [PATCH 73/82] swapping to have defaults expected for pull request --- .github/workflows/compare_post_benchmark.yaml | 15 ++++++++------- tests/test_run_ref.py | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare_post_benchmark.yaml index 9e75dea2..86507aa1 100644 --- a/.github/workflows/compare_post_benchmark.yaml +++ b/.github/workflows/compare_post_benchmark.yaml @@ -1,4 +1,4 @@ -name: "Post benchmark results" +name: "Run Compare Post Benchmark" on: [push, pull_request] #on: # workflow_run: @@ -172,15 +172,16 @@ jobs: json.dump(net_json, f) # Print github event_name and ref_name and the boolean check for whether gh-pages should be updated - - name: Print things + - name: Print Event, Ref, and Upload Boolean run: | echo "Event Name: ${{ github.event_name }}" echo "Ref Name: ${{ github.ref_name }}" - echo "Update gh-pages: ${{ github.event_name == 'push' && github.ref_name == '1.1_ref_sim_ci_workflow' }}" + echo "Update gh-pages: ${{ github.event_name == 'push' && github.ref_name == 'main' }}" # Compares the data from the specified "output-file-path" and compares # with the latest data from the gh-pages branch. If performance regression - # occurs, fails the test and alerts. If no performance regression, (TODO: FIXME: Currently still comments, but will only comment in final version if performance regression has occurred) + # occurs, fails the test and alerts. Will only comment in if performance + # regression has occurred. # NOTE: it is important that this does not modify gh-pages on pull request # https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#caveats # This only updates gh-pages if a push to main occurs @@ -201,8 +202,8 @@ jobs: # Enable Job Summary for PRs summary-always: true # Always leave a comment - comment-always: true + #comment-always: true github-token: ${{ secrets.GITHUB_TOKEN }} # Push and deploy GitHub pages branch automatically - auto-push: ${{ github.event_name == 'push' && github.ref_name == '1.1_ref_sim_ci_workflow' }} - save-data-file: ${{ github.event_name == 'push' && github.ref_name == '1.1_ref_sim_ci_workflow' }} + auto-push: ${{ github.event_name == 'push' && github.ref_name == 'main' }} + save-data-file: ${{ github.event_name == 'push' && github.ref_name == 'main' }} diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 46ead372..75f7a9f1 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -145,8 +145,8 @@ def download_sim(target_dir, sim_name): def compare_uvh5(uv_ref, uv_new): # takes as input two UVData objects, and computes relevant quantities for determining how # similar the data are. Prints the histories before setting them equal. Currently only runs - # an equality check but (TODO: FIXME) should make a much more exhaustive check OR just turn - # back on the exact check and update the sim output when it differs (Do this tbh) + # an equality check but should make a much more exhaustive check OR just turn + # back on the exact check and update the sim output when it differs import numpy as np # print histories From 3d55832fefee53d7cdf737442b40319e8f40db73 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Wed, 4 Dec 2024 23:53:28 -0500 Subject: [PATCH 74/82] changed underscore to hyphen to match style --- .../{compare_post_benchmark.yaml => compare-post-benchmark.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{compare_post_benchmark.yaml => compare-post-benchmark.yaml} (100%) diff --git a/.github/workflows/compare_post_benchmark.yaml b/.github/workflows/compare-post-benchmark.yaml similarity index 100% rename from .github/workflows/compare_post_benchmark.yaml rename to .github/workflows/compare-post-benchmark.yaml From 414c5d7d9d61753e42b07e639ff8fc33806ebf4f Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 9 Dec 2024 14:30:39 -0500 Subject: [PATCH 75/82] Tentative README update -- should probably add a section of regression testing / ci in the developer section of the docs, and amend README to link to it --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9b63222e..daf025ba 100644 --- a/README.md +++ b/README.md @@ -127,9 +127,21 @@ One other package, pytest-xdist, is not required, but can be used to speed up ru the test suite by running tests in parallel. To use it call pytest with the ```-n auto``` option. -Two additional packages, pytest-benchmark and requests, are required if one needs to locally run +Two additional packages, pytest-benchmark and requests, are required if you need to locally run single core regression testing of the reference simulations. For more realistic benchmarking at any level of scale, see [Benchmarking](https://pyuvsim.readthedocs.io/en/latest/developers.html#benchmarking). +To run a single core regression test of the reference simulations, you need to specify a reference +simulation with the ```refsim``` flag and use ```benchmark-only```. Additionally, use mpiexec to +run pytest as follows: +``` + mpiexec -n 1 -np 1 pytest --refsim=1.1_uniform --benchmark-only +``` +, where 1.1_uniform is the specific reference simulation being tested. You can use the ```refsim``` +flag multiple times -- +``` +--refsim=refsim1 --refsim=refsim2 +``` +-- to parametrize multiple reference simulations. One way to ensure you have all the needed packages is to use the included `environment.yaml` file to create a new environment that will From d8ce0fc134f042c1d9d4204abd11a8b8516c310f Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 9 Dec 2024 14:47:13 -0500 Subject: [PATCH 76/82] made data comparison same as np.testing.assert_allclose defaults, removed some commented out code and comments --- tests/test_run_ref.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/test_run_ref.py b/tests/test_run_ref.py index 75f7a9f1..679b853d 100644 --- a/tests/test_run_ref.py +++ b/tests/test_run_ref.py @@ -144,9 +144,7 @@ def download_sim(target_dir, sim_name): def compare_uvh5(uv_ref, uv_new): # takes as input two UVData objects, and computes relevant quantities for determining how - # similar the data are. Prints the histories before setting them equal. Currently only runs - # an equality check but should make a much more exhaustive check OR just turn - # back on the exact check and update the sim output when it differs + # similar the data are. Prints the histories before setting them equal. import numpy as np # print histories @@ -175,9 +173,13 @@ def compare_uvh5(uv_ref, uv_new): frac_diff[np.isnan(frac_diff)] = 0 max_relative_diff = np.amax(frac_diff) + # using np.testing.assert_allclose defaults for comparison + rtol = 1e-7 + atol = 0 + # generate a true/false ndarray for passing and failing cases # should match output of np.testing.assert_allclose - cases = np.abs(new_arr - ref_arr) <= (1e-8 + 1e-5 * np.abs(ref_arr)) + cases = np.abs(new_arr - ref_arr) <= (atol + rtol * np.abs(ref_arr)) outcome = cases.all() # get unique outcomes (true / false) and corresponding counts @@ -209,11 +211,6 @@ def compare_uvh5(uv_ref, uv_new): # simulation output assert uv_new == uv_ref - # fail if any element of data_array is different between new and ref - # currently commented out but might be worth implementing and updating ref_sim file each time - # the assertion fails for the relevant files - # assert (ref_arr == new_arr).all() - def construct_filepaths(target_dir, sim): # takes as input the sim name (NEEDS TO BE AN EXISTING SIM IN THE DATA DIRECTORY), then From 16f46595bdab0171703bab26fff6e6bae0b57dee Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 9 Dec 2024 14:51:07 -0500 Subject: [PATCH 77/82] fixed typos in ci workflow --- .github/workflows/compare-post-benchmark.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/compare-post-benchmark.yaml b/.github/workflows/compare-post-benchmark.yaml index 86507aa1..651ed15d 100644 --- a/.github/workflows/compare-post-benchmark.yaml +++ b/.github/workflows/compare-post-benchmark.yaml @@ -35,14 +35,7 @@ jobs: # all jobs should run in parallel matrix: include: - - id: 1.1_uniform - - id: 1.1_gauss - - id: 1.1_mwa - - id: 1.2_uniform - - id: 1.2_gauss - - id: 1.3_uniform - - id: 1.3_gauss - + - id: [1.1_uniform, 1.1_gauss, 1.1_mwa, 1.2_uniform, 1.2_gauss, 1.3_uniform, 1.3_gauss] defaults: run: # Adding -l {0} helps ensure conda can be found properly. From 01739c9598b9f36d5b39a8c06feaf668bb8961b0 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 9 Dec 2024 14:54:51 -0500 Subject: [PATCH 78/82] fixed formatting for a line --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index daf025ba..42779092 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ To run a single core regression test of the reference simulations, you need to s simulation with the ```refsim``` flag and use ```benchmark-only```. Additionally, use mpiexec to run pytest as follows: ``` - mpiexec -n 1 -np 1 pytest --refsim=1.1_uniform --benchmark-only +mpiexec -n 1 -np 1 pytest --refsim=1.1_uniform --benchmark-only ``` , where 1.1_uniform is the specific reference simulation being tested. You can use the ```refsim``` flag multiple times -- From 32fa583beba1e263765524f23c969c7b76db0136 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 9 Dec 2024 14:57:45 -0500 Subject: [PATCH 79/82] Futher updated the README --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 42779092..e96a3414 100644 --- a/README.md +++ b/README.md @@ -136,12 +136,8 @@ run pytest as follows: ``` mpiexec -n 1 -np 1 pytest --refsim=1.1_uniform --benchmark-only ``` -, where 1.1_uniform is the specific reference simulation being tested. You can use the ```refsim``` -flag multiple times -- -``` ---refsim=refsim1 --refsim=refsim2 -``` --- to parametrize multiple reference simulations. +1.1_uniform would be the specific reference simulation being tested. You can use the ```refsim``` +flag multiple times to parametrize multiple reference simulations: ```--refsim=refsim1 --refsim=refsim2```. One way to ensure you have all the needed packages is to use the included `environment.yaml` file to create a new environment that will From 031fbeb7b9ceb9b423f9f351797f7f9bef7dfd51 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 9 Dec 2024 15:14:02 -0500 Subject: [PATCH 80/82] switching back to multiple ids --- .github/workflows/compare-post-benchmark.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/compare-post-benchmark.yaml b/.github/workflows/compare-post-benchmark.yaml index 651ed15d..86507aa1 100644 --- a/.github/workflows/compare-post-benchmark.yaml +++ b/.github/workflows/compare-post-benchmark.yaml @@ -35,7 +35,14 @@ jobs: # all jobs should run in parallel matrix: include: - - id: [1.1_uniform, 1.1_gauss, 1.1_mwa, 1.2_uniform, 1.2_gauss, 1.3_uniform, 1.3_gauss] + - id: 1.1_uniform + - id: 1.1_gauss + - id: 1.1_mwa + - id: 1.2_uniform + - id: 1.2_gauss + - id: 1.3_uniform + - id: 1.3_gauss + defaults: run: # Adding -l {0} helps ensure conda can be found properly. From ca2c3c7eadd3e07040f096ba0bdf78de1ff3022b Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Mon, 9 Dec 2024 15:53:16 -0500 Subject: [PATCH 81/82] refactored job matrix --- .github/workflows/compare-post-benchmark.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/compare-post-benchmark.yaml b/.github/workflows/compare-post-benchmark.yaml index 86507aa1..0e6655f4 100644 --- a/.github/workflows/compare-post-benchmark.yaml +++ b/.github/workflows/compare-post-benchmark.yaml @@ -34,14 +34,7 @@ jobs: strategy: # all jobs should run in parallel matrix: - include: - - id: 1.1_uniform - - id: 1.1_gauss - - id: 1.1_mwa - - id: 1.2_uniform - - id: 1.2_gauss - - id: 1.3_uniform - - id: 1.3_gauss + id: [1.1_uniform, 1.1_gauss, 1.1_mwa, 1.2_uniform, 1.2_gauss, 1.3_uniform, 1.3_gauss] defaults: run: From ca1e7da246a9b9691fd5b4d3a72cc49334f6ecb1 Mon Sep 17 00:00:00 2001 From: Mitchell Burdorf Date: Thu, 19 Dec 2024 06:04:02 -0500 Subject: [PATCH 82/82] swapped discussion to docs for pytest regression testing --- README.md | 10 +--------- docs/developers.rst | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e96a3414..7c0317fe 100644 --- a/README.md +++ b/README.md @@ -129,15 +129,7 @@ the test suite by running tests in parallel. To use it call pytest with the Two additional packages, pytest-benchmark and requests, are required if you need to locally run single core regression testing of the reference simulations. For more realistic benchmarking at -any level of scale, see [Benchmarking](https://pyuvsim.readthedocs.io/en/latest/developers.html#benchmarking). -To run a single core regression test of the reference simulations, you need to specify a reference -simulation with the ```refsim``` flag and use ```benchmark-only```. Additionally, use mpiexec to -run pytest as follows: -``` -mpiexec -n 1 -np 1 pytest --refsim=1.1_uniform --benchmark-only -``` -1.1_uniform would be the specific reference simulation being tested. You can use the ```refsim``` -flag multiple times to parametrize multiple reference simulations: ```--refsim=refsim1 --refsim=refsim2```. +any level of scale, and for instruction on regression testing with pytest, see [Benchmarking](https://pyuvsim.readthedocs.io/en/latest/developers.html#benchmarking). One way to ensure you have all the needed packages is to use the included `environment.yaml` file to create a new environment that will diff --git a/docs/developers.rst b/docs/developers.rst index 6e19c5b2..8c58bf0c 100644 --- a/docs/developers.rst +++ b/docs/developers.rst @@ -31,6 +31,9 @@ For more details, see `reference_simulations/README.md `_. + +Running a Reference Simulation with pytest-benchmark +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To run a single core regression test of the reference simulations, you need to specify a reference +simulation with the ``refsim`` flag and use ``benchmark-only``. Additionally, you need to use +mpiexec to run pytest as follows: + + .. code-block:: python + + # use mpiexec to run pytest specifying one core + > mpiexec -n 1 -np 1 pytest --refsim=1.1_uniform --benchmark-only + +Here "1.1_uniform" would be the specific reference simulation being tested. You can use the ``refsim`` +flag multiple times to parametrize multiple reference simulations: ``--refsim=refsim1 --refsim=refsim2``. + +We run single core regression tests of the available reference simulations with pytest and pytest-benchmark via our github ci workflow on every push or pull request. We do so to ensure output and runtime consistency. As we only run the simulations with a single core, the benchmarking aspect of these tests is only relevant for linear operations and not a test of any parallelism. + +The available ``refsim`` values are: + +* 1.1_uniform +* 1.1_gauss +* 1.1_mwa +* 1.2_uniform +* 1.2_gauss +* 1.3_uniform +* 1.3_gauss