Use pytest (#1113)

tests: use pytest to run in CI; fix some tests

The resources.get_local_file test would fail if a user had a downloaded de421.bsp in their cache.  Adds a download [bool] arg to resources.get_local_file

Co-authored-by: Yilun Guan <[email protected]>

.github/workflows/test.yml

@@ -56,17 +56,19 @@ jobs:
           python3 -m pip install -vvv .[tests]
 
       - name: Run Serial Tests
+        working-directory: ./tests
         run: |
           export OMP_NUM_THREADS=2
           export OPENBLAS_NUM_THREADS=2
           export MPI_DISABLE=1
-          python3 -m unittest discover
+          python3 -m pytest -vv
 
       - name: Run MPI Tests
+        working-directory: ./tests
         run: |
           export OMP_NUM_THREADS=1
           export OPENBLAS_NUM_THREADS=1
-          mpirun -n 2 python3 -m unittest discover
+          mpirun -n 2 python3 -m pytest -vv
 
   # FIXME:  Re-enable after testing this procedure on a local
   # apple machine.

setup.py

@@ -59,6 +59,7 @@
     'skyfield',
     'so3g',
     'pixell',
+    'pytest',
     'scikit-image',
     'pyfftw',
     'numdifftools',

sotodlib/core/resources.py

@@ -3,15 +3,17 @@
 
 from ast import literal_eval
 
+# This is a listing of public URLs from which to grab some useful data
+# files. On shared systems it may be wise to maintain a shared cache of
+# such files -- see get_local_file.
 
 RESOURCE_DEFAULTS = {
     "de421.bsp": "ftp://ssd.jpl.nasa.gov/pub/eph/planets/bsp/de421.bsp",
 }
 
 
-def get_local_file(filename: str, cache: bool = True) -> str:
-    """
-    This function utilizes RESOURCE_DEFAULTS or SOTODLIB_RESOURCES environment
+def get_local_file(filename: str, cache: bool = True, download: bool = True) -> str:
+    """This function utilizes RESOURCE_DEFAULTS or SOTODLIB_RESOURCES environment
     variable to manage resource files such as a planet catalog from NASA.
 
     RESOURCE_DEFAULTS and SOTODLIB_RESOURCES are dictionaries with key a filename
@@ -43,6 +45,8 @@ def get_local_file(filename: str, cache: bool = True) -> str:
       filename: The name of the file to grab
       cache: A boolean indicating that downloaded files should be cached in user
               home folder (~/.sotodlib/filecache/).
+      download: If False, the file will not be downloaded even if
+        config specifies a URL and the local cache does not have the file.
 
     Returns:
       The absolute path of the file.
@@ -51,6 +55,7 @@ def get_local_file(filename: str, cache: bool = True) -> str:
         RuntimeError: when the requested resource file does not exist as a key
                       in SOTODLIB_RESOURCES env variable or RESOURCE_DEFAULTS.
         RuntimeErorr: When the value of a key is not an ftp or file path.
+
     """
 
     # Local cache. This is per user, however we may want to try and right in
@@ -83,7 +88,8 @@ def get_local_file(filename: str, cache: bool = True) -> str:
         target_path = local_cache if cache else "/tmp/"
         os.makedirs(name=target_path, exist_ok=True)
         target_file = os.path.join(target_path, filename)
-        _, headers = urlretrieve(de_url, target_file)
+        if download:
+            _, headers = urlretrieve(de_url, target_file)
     elif de_url.startswith("file://"):
         target_file = de_url[7:]
     else:

tests/test_core_resources.py

@@ -2,33 +2,35 @@
 import unittest
 import json
 import shutil
-import pytest
 
-from sotodlib.core.resources import get_local_file
+from sotodlib.core import resources
+
+ODD_NAME = 'planet47-ephemeris.txt'
 
 
 class TestCoreResources(unittest.TestCase):
 
     def test_get_local_file(self):
 
-        # t = get_local_file("de421.bsp", cache=True)
-        # expected_path = os.path.join(
-        #     os.path.expanduser("~"), ".sotodlib/filecache/de421.bsp"
-        # )
-        # self.assertEqual(expected_path, t)
-        # shutil.rmtree(os.path.join(os.path.expanduser("~"), ".sotodlib/"))
+        # This url should not be accessed, as long as download=False
+        resources.RESOURCE_DEFAULTS[ODD_NAME] = \
+            'ftp://simonsobs.org/sotodlib-ci.txt'
+
         os.environ["SOTODLIB_RESOURCES"] = json.dumps(
             {"someotherfile": "file://somepath/otherfile"}
         )
-        t = get_local_file("de421.bsp", cache=False)
-        expected_path = "/tmp/de421.bsp"
+
+        # Use a fake file for this, or else it might find a
+        # user-cached copy.
+        t = resources.get_local_file(ODD_NAME, cache=False, download=False)
+        expected_path = "/tmp/" + ODD_NAME
         self.assertEqual(expected_path, t)
 
         os.environ["SOTODLIB_RESOURCES"] = json.dumps(
-            {"de421.bsp": "file://somepath/de421.bsp"}
+            {ODD_NAME: "file://somepath/de421.bsp"}
         )
-        t = get_local_file("de421.bsp")
+        t = resources.get_local_file(ODD_NAME)
         self.assertEqual("somepath/de421.bsp", t)
 
-        with pytest.raises(RuntimeError):
-            get_local_file("doesnotexist.file")
+        with self.assertRaises(RuntimeError):
+            resources.get_local_file("doesnotexist.file")

tests/test_imprinter.py

@@ -8,27 +8,48 @@
 from sotodlib.io.imprinter import *
 from sotodlib.io.g3tsmurf_db import Files
 
+from ._helpers import mpi_multi
+
+
+# These tests are serial-only.
+if mpi_multi():
+    pytest.skip(allow_module_level=True)
+
+
 @pytest.fixture(scope="session", autouse=True)
 def imprinter():
     with tempfile.NamedTemporaryFile(mode='w') as f:
+        g3tsmurf_file = tempfile.NamedTemporaryFile(mode='w', delete=False)
+        g3tsmurf_config = {
+            "data_prefix": "/data",
+            "db_path": "test_g3tsmurf.db"
+        }
+        yaml.dump(g3tsmurf_config, g3tsmurf_file)
+        g3tsmurf_file.close()
+
         im_config = {
             'db_path': '_pytest_imprinter.db',
-            'sources': {
+            'g3tsmurf': g3tsmurf_file.name,
+            'output_root': 'test_output',
+            'tel_tubes': {
                 'lat': {
                     'slots': ['slot1', 'slot2']
                 },
             }
         }
         yaml.dump(im_config, f)
         f.flush()
-        im = Imprinter(f.name)
+        im = Imprinter(f.name, make_db=True)
     return im
 
 # use this to clean up db file afterwards
 @pytest.fixture(scope="session", autouse=True)
 def delete_imprinter_db(imprinter):
     yield
-    os.remove(imprinter.db_path)
+    if os.path.exists(imprinter.db_path):
+        os.remove(imprinter.db_path)
+    if os.path.exists('test_g3tsmurf.db'):
+        os.remove('test_g3tsmurf.db')
 
 @pytest.fixture
 def obsset():
@@ -41,12 +62,15 @@ def obsset():
     file.start = dt_t0
     file.stop = dt_t1
     file.n_channels = 10
-    obs1 = create_autospec(G3tObservations)
-    obs1.obs_id = f"oper_slot1_{t0}"
-    obs1.files = [file]*2
-    obs2 = create_autospec(G3tObservations)
+
+    obs1 = create_autospec(spec=G3tObservations, instance=True)
+    obs1.obs_id = f"oper_slot1_{t0}" 
+    obs1.files = [file] * 2
+    obs1.timing = True
+    obs2 = create_autospec(spec=G3tObservations, instance=True)
     obs2.obs_id = f"oper_slot2_{t0}"
-    obs2.files = [file]*2
+    obs2.files = [file] * 2
+    obs2.timing = True
     obsset = ObsSet([obs1, obs2], mode="oper", slots=["slot1", "slot2", "slot3"], tel_tube="lat")
     return obsset
 
@@ -68,12 +92,16 @@ def test_register_book(imprinter, obsset):
     assert book.obs[1].obs_id == "oper_slot2_1674090159"
     assert book.tel_tube == "lat"
     assert book.type == "oper"
-    assert book.start == datetime.datetime.fromtimestamp(
-        1674090159, tz=datetime.timezone.utc
+    expected_start = (
+        datetime.datetime.fromtimestamp(1674090159, tz=datetime.timezone.utc)
+        .replace(tzinfo=None)
     )
-    assert book.stop == datetime.datetime.fromtimestamp(
-        1674090259, tz=datetime.timezone.utc
+    expected_stop = (
+        datetime.datetime.fromtimestamp(1674090259, tz=datetime.timezone.utc)
+        .replace(tzinfo=None)
     )
+    assert book.start == expected_start
+    assert book.stop == expected_stop
     assert book.max_channels == 10
     assert book.message == ""
     assert book.slots == "slot1,slot2"