update API usage in tests, along with reference data (#8)

* update test to check generated configuration against reference

* fix auto-formatted reference file

* fix download

* remove reference files with absolute paths

* minor fixes and `nemspy` API updates

* minor fixes and `nemspy` API updates

* comment out spinup start and end dates

* update examples to use new constructor

* remove version pin

* only extract specific files

* add print debugging

* fix string

* remove `atm_namelist.rc.*` as it is just a symlink

* unformat

* fix partial run

.gitignore

@@ -128,4 +128,7 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-.idea/
+.idea/
+/examples/data
+/tests/data/*/output/*
+/tests/data/*/input/*

coupledmodeldriver/adcirc.py

@@ -5,15 +5,13 @@
 from os import PathLike
 from pathlib import Path
 import re
-import tarfile
 
 from adcircpy import AdcircMesh, AdcircRun
 from adcircpy.forcing.base import Forcing
 from adcircpy.server import SlurmConfig
 from nemspy import ModelingSystem
 from nemspy.model import ADCIRCEntry
 import numpy
-import requests
 
 from .job_script import EnsembleSlurmScript, Platform, SlurmEmailType
 from .utilities import get_logger
@@ -32,7 +30,6 @@ def write_adcirc_configurations(
     email_address: str = None,
     wall_clock_time: timedelta = None,
     spinup: timedelta = None,
-    source_filename: PathLike = None,
     forcings: [Forcing] = None,
 ):
     """
@@ -78,20 +75,17 @@ def write_adcirc_configurations(
     if wall_clock_time is None:
         wall_clock_time = timedelta(minutes=30)
 
-    errors = []
-    fort13_filename = mesh_directory / 'fort.14'
-    fort14_filename = mesh_directory / 'fort.14'
+    fort13_filename = mesh_directory / 'fort.13'
     if not fort13_filename.exists():
-        errors.append(f'mesh values (nodal attributes) not found at {fort13_filename}')
+        LOGGER.warning(f'mesh values (nodal attributes) not found at {fort13_filename}')
+    fort14_filename = mesh_directory / 'fort.14'
     if not fort14_filename.exists():
-        errors.append(f'mesh XY not found at {fort14_filename}')
-    if len(errors) > 0:
-        raise FileNotFoundError('; '.join(errors))
+        raise FileNotFoundError(f'mesh XY not found at {fort14_filename}')
 
     if spinup is not None and isinstance(spinup, timedelta):
         spinup = ModelingSystem(
             nems.start_time - spinup,
-            spinup,
+            nems.start_time,
             nems.interval,
             ocn=copy.deepcopy(nems['OCN']),
             **nems.attributes,
@@ -177,26 +171,27 @@ def write_adcirc_configurations(
     driver = AdcircRun(
         mesh=mesh,
         start_date=nems.start_time,
-        end_date=nems.start_time + nems.duration,
+        end_date=nems.end_time,
         spinup_time=timedelta(days=5),
         server_config=slurm,
     )
 
-    spinup_start = spinup.start_time if spinup is not None else None
-    spinup_end = spinup.start_time + spinup.duration if spinup is not None else None
+    # spinup_start = spinup.start_time if spinup is not None else None
+    # spinup_end = spinup.end_time if spinup is not None else None
     spinup_interval = spinup.interval if spinup is not None else None
+
     stations_filename = mesh_directory / 'stations.txt'
     if stations_filename.exists():
         driver.import_stations(stations_filename)
-        driver.set_elevation_stations_output(nems.interval, spinup=spinup_interval,
-                                             spinup_start=spinup_start, spinup_end=spinup_end)
-        driver.set_velocity_stations_output(nems.interval, spinup=spinup_interval,
-                                            spinup_start=spinup_start, spinup_end=spinup_end)
+        driver.set_elevation_stations_output(nems.interval, spinup=spinup_interval)
+        # spinup_start=spinup_start, spinup_end=spinup_end)
+        driver.set_velocity_stations_output(nems.interval, spinup=spinup_interval)
+        # spinup_start=spinup_start, spinup_end=spinup_end)
 
-    driver.set_elevation_surface_output(nems.interval, spinup=spinup_interval,
-                                        spinup_start=spinup_start, spinup_end=spinup_end)
-    driver.set_velocity_surface_output(nems.interval, spinup=spinup_interval,
-                                       spinup_start=spinup_start, spinup_end=spinup_end)
+    driver.set_elevation_surface_output(nems.interval, spinup=spinup_interval)
+    # spinup_start=spinup_start, spinup_end=spinup_end)
+    driver.set_velocity_surface_output(nems.interval, spinup=spinup_interval)
+    # spinup_start=spinup_start, spinup_end=spinup_end)
 
     for run_name, (value, attribute_name) in runs.items():
         run_directory = output_directory / run_name
@@ -225,8 +220,7 @@ def write_adcirc_configurations(
 
     for filename in coldstart_filenames + [atm_namelist_filename]:
         coldstart_filename = Path(f'{filename}.coldstart')
-        if coldstart_filename.exists():
-            os.remove(coldstart_filename)
+        os.remove(coldstart_filename)
         filename.rename(coldstart_filename)
 
     if spinup is not None:
@@ -236,8 +230,7 @@ def write_adcirc_configurations(
 
     for filename in hotstart_filenames + [atm_namelist_filename]:
         hotstart_filename = Path(f'{filename}.hotstart')
-        if hotstart_filename.exists():
-            os.remove(hotstart_filename)
+        os.remove(hotstart_filename)
         filename.rename(hotstart_filename)
 
     pattern = re.compile(' p*adcirc')
@@ -253,25 +246,3 @@ def write_adcirc_configurations(
             text = re.sub(pattern, replacement, text)
             with open(job_filename, 'w') as job_file:
                 job_file.write(text)
-
-
-def download_shinnecock_ike_mesh(directory: str):
-    """
-    fetch shinnecock inlet test data
-    :param directory: local directory
-    """
-
-    if not isinstance(directory, Path):
-        directory = Path(directory)
-
-    if not directory.exists():
-        directory.mkdir(parents=True, exist_ok=True)
-
-    url = 'https://www.dropbox.com/s/1wk91r67cacf132/NetCDF_shinnecock_inlet.tar.bz2?dl=1'
-    remote_file = requests.get(url, stream=True)
-    temporary_filename = directory / 'temp.tar.gz'
-    with open(temporary_filename, 'b+w') as local_file:
-        local_file.write(remote_file.raw.read())
-    with tarfile.open(temporary_filename, 'r:bz2') as local_file:
-        local_file.extractall(directory)
-    os.remove(temporary_filename)

examples/hera/hera_hsofs_irma.py

@@ -30,7 +30,7 @@
 
     nems = ModelingSystem(
         start_time=datetime(2017, 9, 5),
-        duration=timedelta(days=14.5),
+        end_time=datetime(2017, 9, 5) + timedelta(days=14.5),
         interval=timedelta(hours=1),
         atm=AtmosphericMeshEntry(FORCINGS_DIRECTORY / 'Wind_HWRF_IRMA_Nov2018_ExtendedSmoothT.nc'),
         wav=WaveMeshEntry(FORCINGS_DIRECTORY / 'ww3.HWRF.NOV2018.2017_sxy.nc'),

examples/hera/hera_hsofs_sandy.py

@@ -30,7 +30,7 @@
 
     nems = ModelingSystem(
         start_time=datetime(2012, 10, 22, 6),
-        duration=timedelta(days=14.5),
+        end_time=datetime(2012, 10, 22, 6) + timedelta(days=14.5),
         interval=timedelta(hours=1),
         atm=AtmosphericMeshEntry(FORCINGS_DIRECTORY / 'SANDY_HWRF_HSOFS_Nov2018.nc'),
         wav=WaveMeshEntry(FORCINGS_DIRECTORY / 'ww3.HWRF.NOV2018.2012_sxy.nc'),

examples/hera/hera_shinnecock_ike.py

@@ -20,7 +20,7 @@
 OUTPUT_DIRECTORY = (Path(__file__).parent / '../data') / 'configuration' / 'hera' / 'shinnecock' / 'ike'
 
 if __name__ == '__main__':
-    runs = {f'nems_shinnecock_test': (None, None)}
+    runs = {f'test_case_1': (None, None)}
 
     # init tidal forcing and setup requests
     tidal_forcing = Tides()
@@ -30,7 +30,7 @@
 
     nems = ModelingSystem(
         start_time=datetime(2008, 8, 23),
-        duration=timedelta(days=14.5),
+        end_time=datetime(2008, 8, 23) + timedelta(days=14.5),
         interval=timedelta(hours=1),
         atm=AtmosphericMeshEntry(FORCINGS_DIRECTORY / 'wind_atm_fin_ch_time_vec.nc'),
         wav=WaveMeshEntry(FORCINGS_DIRECTORY / 'ww3.Constant.20151214_sxy_ike_date.nc'),
@@ -52,7 +52,7 @@
         runs,
         MESH_DIRECTORY,
         OUTPUT_DIRECTORY,
-        name='nems_shinnecock_test',
+        name='test_case_1',
         email_address='[email protected]',
         platform=Platform.HERA,
         spinup=timedelta(days=12.5),

examples/hera/hera_shinnecock_ike_perturbed_mannings_n.py

@@ -40,7 +40,7 @@
 
     nems = ModelingSystem(
         start_time=datetime(2008, 8, 23),
-        duration=timedelta(days=14.5),
+        end_time=datetime(2008, 8, 23) + timedelta(days=14.5),
         interval=timedelta(hours=1),
         atm=AtmosphericMeshEntry(FORCINGS_DIRECTORY / 'wind_atm_fin_ch_time_vec.nc'),
         wav=WaveMeshEntry(FORCINGS_DIRECTORY / 'ww3.Constant.20151214_sxy_ike_date.nc'),

examples/local/local_hsofs_irma.py

@@ -30,7 +30,7 @@
 
     nems = ModelingSystem(
         start_time=datetime(2017, 9, 5),
-        duration=timedelta(days=14.5),
+        end_time=datetime(2017, 9, 5) + timedelta(days=14.5),
         interval=timedelta(hours=1),
         atm=AtmosphericMeshEntry(FORCINGS_DIRECTORY / 'Wind_HWRF_IRMA_Nov2018_ExtendedSmoothT.nc'),
         wav=WaveMeshEntry(FORCINGS_DIRECTORY / 'ww3.HWRF.NOV2018.2017_sxy.nc'),

examples/local/local_hsofs_sandy.py

@@ -30,7 +30,7 @@
 
     nems = ModelingSystem(
         start_time=datetime(2012, 10, 22, 6),
-        duration=timedelta(days=14.5),
+        end_time=datetime(2012, 10, 22, 6) + timedelta(days=14.5),
         interval=timedelta(hours=1),
         atm=AtmosphericMeshEntry(FORCINGS_DIRECTORY / 'SANDY_HWRF_HSOFS_Nov2018.nc'),
         wav=WaveMeshEntry(FORCINGS_DIRECTORY / 'ww3.HWRF.NOV2018.2012_sxy.nc'),

examples/local/local_shinnecock_ike.py

@@ -20,7 +20,7 @@
 OUTPUT_DIRECTORY = (Path(__file__).parent / '../data') / 'configuration' / 'local' / 'hsofs' / 'ike'
 
 if __name__ == '__main__':
-    runs = {f'nems_shinnecock_test': (None, None)}
+    runs = {f'test_case_1': (None, None)}
 
     # init tidal forcing and setup requests
     tidal_forcing = Tides()
@@ -30,7 +30,7 @@
 
     nems = ModelingSystem(
         start_time=datetime(2008, 8, 23),
-        duration=timedelta(days=14.5),
+        end_time=datetime(2008, 8, 23) + timedelta(days=14.5),
         interval=timedelta(hours=1),
         atm=AtmosphericMeshEntry(FORCINGS_DIRECTORY / 'wind_atm_fin_ch_time_vec.nc'),
         wav=WaveMeshEntry(FORCINGS_DIRECTORY / 'ww3.Constant.20151214_sxy_ike_date.nc'),
@@ -52,7 +52,7 @@
         runs,
         MESH_DIRECTORY,
         OUTPUT_DIRECTORY,
-        name='nems_shinnecock_test',
+        name='test_case_1',
         email_address='[email protected]',
         platform=Platform.LOCAL,
         spinup=timedelta(days=12.5),

examples/local/local_shinnecock_ike_perturbed_mannings_n.py

@@ -40,7 +40,7 @@
 
     nems = ModelingSystem(
         start_time=datetime(2008, 8, 23),
-        duration=timedelta(days=14.5),
+        end_time=datetime(2008, 8, 23) + timedelta(days=14.5),
         interval=timedelta(hours=1),
         atm=AtmosphericMeshEntry(FORCINGS_DIRECTORY / 'wind_atm_fin_ch_time_vec.nc'),
         wav=WaveMeshEntry(FORCINGS_DIRECTORY / 'ww3.Constant.20151214_sxy_ike_date.nc'),

examples/stampede2/stampede2_hsofs_sandy.py

@@ -30,7 +30,7 @@
 
     nems = ModelingSystem(
         start_time=datetime(2012, 10, 22, 6),
-        duration=timedelta(days=14.5),
+        end_time=datetime(2012, 10, 22, 6) + timedelta(days=14.5),
         interval=timedelta(hours=1),
         atm=AtmosphericMeshEntry(FORCINGS_DIRECTORY / 'SANDY_HWRF_HSOFS_Nov2018.nc'),
         wav=WaveMeshEntry(FORCINGS_DIRECTORY / 'ww3.HWRF.NOV2018.2012_sxy.nc'),

examples/stampede2/stampede2_shinnecock_ike.py

@@ -20,7 +20,7 @@
 OUTPUT_DIRECTORY = (Path(__file__).parent / '../data') / 'configuration' / 'stampede2' / 'shinnecock' / 'ike'
 
 if __name__ == '__main__':
-    runs = {f'nems_shinnecock_test': (None, None)}
+    runs = {f'test_case_1': (None, None)}
 
     # init tidal forcing and setup requests
     tidal_forcing = Tides()
@@ -30,7 +30,7 @@
 
     nems = ModelingSystem(
         start_time=datetime(2008, 8, 23),
-        duration=timedelta(days=14.5),
+        end_time=datetime(2008, 8, 23) + timedelta(days=14.5),
         interval=timedelta(hours=1),
         atm=AtmosphericMeshEntry(FORCINGS_DIRECTORY / 'wind_atm_fin_ch_time_vec.nc'),
         wav=WaveMeshEntry(FORCINGS_DIRECTORY / 'ww3.Constant.20151214_sxy_ike_date.nc'),
@@ -52,7 +52,7 @@
         runs,
         MESH_DIRECTORY,
         OUTPUT_DIRECTORY,
-        name='nems_shinnecock_test',
+        name='test_case_1',
         email_address='[email protected]',
         platform=Platform.STAMPEDE2,
         spinup=timedelta(days=12.5),

examples/stampede2/stampede2_shinnecock_ike_perturbed_mannings_n.py

@@ -40,7 +40,7 @@
 
     nems = ModelingSystem(
         start_time=datetime(2008, 8, 23),
-        duration=timedelta(days=14.5),
+        end_time=datetime(2008, 8, 23) + timedelta(days=14.5),
         interval=timedelta(hours=1),
         atm=AtmosphericMeshEntry(FORCINGS_DIRECTORY / 'wind_atm_fin_ch_time_vec.nc'),
         wav=WaveMeshEntry(FORCINGS_DIRECTORY / 'ww3.Constant.20151214_sxy_ike_date.nc'),

setup.py

@@ -16,7 +16,7 @@
     try:
         import gdal
     except ImportError:
-        subprocess.check_call([sys.executable, '-m', 'pipwin', 'install', 'gdal==3.1.4'])
+        subprocess.check_call([sys.executable, '-m', 'pipwin', 'install', 'gdal'])
 
     try:
         import fiona
@@ -54,7 +54,7 @@
     packages=find_packages(),
     python_requires='>=3.6',
     setup_requires=['dunamai', 'setuptools>=41.2'],
-    install_requires=['adcircpy', 'nemspy', 'numpy', 'requests'],
+    install_requires=['adcircpy==1.0.16', 'nemspy>=0.6.2', 'numpy', 'requests'],
     extras_require={
         'testing': ['flake8', 'pytest', 'pytest-cov'],
         'development': ['oitnb']

tests/data/test_shinnecock_configuration/reference/model_configure.coldstart

@@ -0,0 +1,14 @@
+# `model_configure` generated with NEMSpy 0.2.2.post14.dev0+21597a1
+total_member:            1
+print_esmf:              .true.
+namelist:                atm_namelist
+PE_MEMBER01:             382
+start_year:              2008
+start_month:             8
+start_day:               10
+start_hour:              12
+start_minute:            0
+start_second:            0
+nhours_fcst:             300
+RUN_CONTINUE:            .false.
+ENS_SPS:                 .false.

tests/data/test_shinnecock_configuration/reference/model_configure.hotstart

@@ -0,0 +1,14 @@
+# `model_configure` generated with NEMSpy 0.2.2.post14.dev0+21597a1
+total_member:            1
+print_esmf:              .true.
+namelist:                atm_namelist
+PE_MEMBER01:             384
+start_year:              2008
+start_month:             8
+start_day:               23
+start_hour:              0
+start_minute:            0
+start_second:            0
+nhours_fcst:             348
+RUN_CONTINUE:            .false.
+ENS_SPS:                 .false.

tests/data/test_shinnecock_configuration/reference/nems.configure.coldstart

@@ -0,0 +1,20 @@
+# `nems.configure` generated with NEMSpy 0.2.2.post14.dev0+21597a1
+# EARTH #
+EARTH_component_list: OCN
+EARTH_attributes::
+  Verbosity = off
+::
+
+# OCN #
+OCN_model:                      adcirc
+OCN_petlist_bounds:             0 381
+OCN_attributes::
+  Verbosity = off
+::
+
+# Run Sequence #
+runSeq::
+  @3600
+    OCN
+  @
+::