Merge branch 'main' into 10-schemaedit

.github/workflows/build-conda.yml

@@ -0,0 +1,20 @@
+name: build_conda
+on:
+  pull_request:
+    branches:
+    # for testing conda build w no upload during PRs
+    - main
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container:
+      image: continuumio/miniconda3:latest
+    steps:
+    - name: Checkout Files
+      uses: actions/checkout@v4
+    - name: Run Docker to Build
+      run: |
+        conda config --append channels conda-forge
+        conda config --append channels noaa-gfdl
+        conda install conda-build conda-verify
+        conda build .

.github/workflows/conda-env-create-run-pytest.yml

@@ -0,0 +1,76 @@
+name: conda-env-create-run-pytest
+
+on: [pull_request]
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 5
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+
+    - name: Add conda to system path
+      run: |
+        # $CONDA is an environment variable pointing to the root of the miniconda directory
+        echo $CONDA/bin >> $GITHUB_PATH
+
+    - name: Create catalogbuilder conda environment
+      run: |
+        # create environment catalogbuilder will be installed into
+        conda env create -f environment.yml --name catalogbuilder
+        # put conda env binaries in GITHUB_PATH
+        echo $CONDA/envs/catalogbuilder/bin >> $GITHUB_PATH
+        # install catalogbuilder to conda env directories
+        $CONDA/envs/catalogbuilder/bin/python -m pip install --prefix $CONDA/envs/catalogbuilder .
+
+    - name: Run pytest in catalogbuilder conda environment
+      run: |
+        which python
+        python --version
+        $CONDA/envs/catalogbuilder/bin/python --version
+        # which pytest
+        $CONDA/envs/catalogbuilder/bin/pytest catalogbuilder
+
+    - name: Make sample data
+      run: |
+        which python
+        python catalogbuilder/tests/make_sample_data.py
+
+    - name: Generate catalog
+      run: |
+        which python
+        python catalogbuilder/scripts/gen_intake_gfdl.py archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp gfdl_autotest
+
+    - name: Generate catalog with yaml
+      run: |
+        which python
+        python catalogbuilder/scripts/gen_intake_gfdl.py --config catalogbuilder/tests/test_config.yaml
+  
+    - name: upload-artifacts
+      uses: actions/upload-artifact@v4
+      with:
+          name: workflow-artifacts1 
+          path: |
+            gfdl_autotest.csv
+            gfdl_autotest.json
+            cats/gfdl_autotest_from_yaml.csv
+            cats/gfdl_autotest_from_yaml.json
+
+    - name: Download all workflow run artifacts
+      uses: actions/download-artifact@v4
+
+    - name: Run pytest with downloaded artifacts
+      run: |
+        $CONDA/envs/catalogbuilder/bin/pytest -v --runxfail catalogbuilder
+
+    - name: Test for completeness
+      run: | 
+        which python
+        python catalogbuilder/scripts/test_catalog.py -tf gfdl_autotest.json catalogbuilder/cats/gfdl_template.json
+        python catalogbuilder/scripts/test_catalog.py -tf catalogbuilder/cats/gfdl_autotest_from_yaml.json

.github/workflows/documentation.yml

@@ -11,12 +11,15 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v3
+
       - name: Install dependencies
         run: |
           pip install sphinx sphinx_rtd_theme myst_parser piccolo_theme
+          
       - name: Sphinx build
         run: |
           sphinx-build doc _build
+          
       - name: Deploy to GitHub Pages
         uses: peaceiris/actions-gh-pages@v3
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}

.github/workflows/publish-conda.yml

@@ -1,11 +1,11 @@
 name: publish_conda
 on:
-  pull_request:
+  push:
     branches:
-    #for testing the restructuring, back on push only later.
+    # only upload upon MR acceptance.
     - main
 jobs:
-  build_and_publish:
+  publish:
     runs-on: ubuntu-latest
     container:
       image: continuumio/miniconda3:latest

MANIFEST.in

@@ -1,2 +1,2 @@
-recursive-include cats *
+recursive-include catalogbuilder/cats *
 

catalogbuilder/intakebuilder/CSVwriter.py

@@ -1,7 +1,8 @@
 import os.path
 import csv
 from csv import writer
-from intakebuilder import builderconfig, configparser 
+#from intakebuilder import builderconfig, configparser
+from . import builderconfig, configparser 
 
 def getHeader(configyaml):
     '''

catalogbuilder/intakebuilder/getinfo.py

@@ -4,7 +4,8 @@
 from csv import writer
 import os
 import xarray as xr
-from intakebuilder import builderconfig, configparser 
+#from intakebuilder import builderconfig, configparser
+from . import builderconfig, configparser 
 
 
 '''

catalogbuilder/intakebuilder/gfdlcrawler.py

@@ -1,5 +1,6 @@
 import os
-from intakebuilder import getinfo, builderconfig
+#from intakebuilder import getinfo, builderconfig
+from . import getinfo, builderconfig
 import sys
 import re
 import operator as op

catalogbuilder/intakebuilder/localcrawler.py

@@ -1,5 +1,6 @@
 import os
-from intakebuilder import getinfo
+#from intakebuilder import getinfo
+from . import getinfo
 import re
 '''
 localcrawler crawls through the local file path, then calls helper functions in the package to getinfo.

catalogbuilder/intakebuilder/s3crawler.py

@@ -1,7 +1,8 @@
 import re
 import boto3
 import botocore
-from intakebuilder import getinfo
+#from intakebuilder import getinfo
+from . import getinfo
 
 '''
 s3 crawler crawls through the S3 bucket, passes the bucket path to the helper functions to getinfo.

catalogbuilder/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+testpaths =
+    catalogbuilder/tests

catalogbuilder/scripts/gen_intake_gfdl.py

@@ -11,7 +11,8 @@
 logger.setLevel(logging.INFO)
 
 try:
-   from intakebuilder import gfdlcrawler, CSVwriter, builderconfig, configparser
+   #from intakebuilder import gfdlcrawler, CSVwriter, builderconfig, configparser
+   from catalogbuilder.intakebuilder import gfdlcrawler, CSVwriter, builderconfig, configparser
 except ModuleNotFoundError:
     print("The module intakebuilder is not installed. Do you have intakebuilder in your sys.path or have you activated the conda environment with the intakebuilder package in it? ")
     print("Attempting again with adjusted sys.path ")

catalogbuilder/scripts/gen_intake_gfdl_runner.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python
 
 #TODO test after conda pkg is published and make changes as needed 
-from catalogbuilder.scripts import gen_intake_gfdl
+#from catalogbuilder.scripts import gen_intake_gfdl
+from . import gen_intake_gfdl
 import sys
 
 input_path = "/archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp/"

catalogbuilder/scripts/gen_intake_gfdl_runner_config.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python
 
-from catalogbuilder.scripts import gen_intake_gfdl
+#from catalogbuilder.scripts import gen_intake_gfdl
+from . import gen_intake_gfdl
 import sys
 
+# this will break at some point #TODO
 sys.argv = ['input_path','--config', '/home/a1r/github/CatalogBuilder/scripts/configs/config-example.yml']
 print(sys.argv)
 gen_intake_gfdl.main()

catalogbuilder/scripts/gen_intake_local.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 
 import os
-from intakebuilder import localcrawler, CSVwriter
+#from intakebuilder import localcrawler, CSVwriter
+from . import localcrawler, CSVwriter
 import logging
 logger = logging.getLogger('local')
-hdlr = logging.FileHandler('/nbhome/a1r/logs/local.log')
+hdlr = logging.FileHandler('/nbhome/a1r/logs/local.log') #TODO this will break
 logger.addHandler(hdlr)
 logger.setLevel(logging.INFO)
 

catalogbuilder/scripts/gen_intake_s3.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import os
-from intakebuilder import getinfo, s3crawler, CSVwriter
+#from intakebuilder import getinfo, s3crawler, CSVwriter
+from . import getinfo, s3crawler, CSVwriter
 import logging
 logger = logging.getLogger('local')
 hdlr = logging.FileHandler('/Users/ar46/logs/local.log')

tests/test_ci_dynamic.py → catalogbuilder/tests/test_ci_dynamic.py

@@ -32,7 +32,7 @@ def test_loadcat():
   #todo check if its readable etc
   #we are using the dynamically generated csv and json for testing in this routine
   #leveraging GitHub actions CI workflow and manifests and caches
-  catspec = pathlib.Path(os.path.dirname(__file__)).parent / 'workflow-artifacts1/gfdl_autotest.json'
+  catspec = pathlib.Path(os.path.dirname(__file__)).parent / '../workflow-artifacts1/gfdl_autotest.json'
   cat = load_cat((str(catspec)))
   try:
     assert isinstance(cat.df, pd.DataFrame),"test failed"