diff --git a/.github/workflows/create-gfdl-catalog.yml b/.github/workflows/create-gfdl-catalog.yml index 24f05dc..7961673 100644 --- a/.github/workflows/create-gfdl-catalog.yml +++ b/.github/workflows/create-gfdl-catalog.yml @@ -29,10 +29,10 @@ jobs: run: python tests/make_sample_data.py - name: 'Generate catalog' run: | - $CONDA/envs/catalogbuilder/bin/python scripts/gen_intake_gfdl.py archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp gfdl_autotest + $CONDA/envs/catalogbuilder/bin/python catalogbuilder/scripts/gen_intake_gfdl.py archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp gfdl_autotest - name: 'Generate catalog with yaml' run: | - $CONDA/envs/catalogbuilder/bin/python scripts/gen_intake_gfdl.py --config tests/test_config.yaml + $CONDA/envs/catalogbuilder/bin/python catalogbuilder/scripts/gen_intake_gfdl.py --config tests/test_config.yaml - name: upload-artifacts1 uses: actions/upload-artifact@v4 with: @@ -50,5 +50,5 @@ jobs: $CONDA/envs/catalogbuilder/bin/pytest -v --runxfail - name: Test for completeness run: | - $CONDA/envs/catalogbuilder/bin/python scripts/test_catalog.py -tf gfdl_autotest.json cats/gfdl_template.json - $CONDA/envs/catalogbuilder/bin/python scripts/test_catalog.py -tf cats/gfdl_autotest_from_yaml.json + $CONDA/envs/catalogbuilder/bin/python catalogbuilder/scripts/test_catalog.py -tf gfdl_autotest.json catalogbuilder/cats/gfdl_template.json + $CONDA/envs/catalogbuilder/bin/python catalogbuilder/scripts/test_catalog.py -tf catalogbuilder/cats/gfdl_autotest_from_yaml.json diff --git a/.github/workflows/publish-conda.yml b/.github/workflows/publish-conda.yml index 1f70a17..580a588 100644 --- a/.github/workflows/publish-conda.yml +++ b/.github/workflows/publish-conda.yml @@ -1,7 +1,8 @@ name: publish_conda on: - push: + pull_request: branches: + #for testing the restructuring, back on push only later. - main jobs: build_and_publish: diff --git a/catalogbuilder/__init__.py b/catalogbuilder/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/catalogbuilder/__init__.py @@ -0,0 +1 @@ + diff --git a/cats/__init__.py b/catalogbuilder/cats/__init__.py similarity index 100% rename from cats/__init__.py rename to catalogbuilder/cats/__init__.py diff --git a/cats/gfdl_autotest.csv b/catalogbuilder/cats/gfdl_autotest.csv similarity index 100% rename from cats/gfdl_autotest.csv rename to catalogbuilder/cats/gfdl_autotest.csv diff --git a/cats/gfdl_template.json b/catalogbuilder/cats/gfdl_template.json similarity index 100% rename from cats/gfdl_template.json rename to catalogbuilder/cats/gfdl_template.json diff --git a/intakebuilder/CSVwriter.py b/catalogbuilder/intakebuilder/CSVwriter.py similarity index 100% rename from intakebuilder/CSVwriter.py rename to catalogbuilder/intakebuilder/CSVwriter.py diff --git a/intakebuilder/__init__.py b/catalogbuilder/intakebuilder/__init__.py similarity index 100% rename from intakebuilder/__init__.py rename to catalogbuilder/intakebuilder/__init__.py diff --git a/intakebuilder/builderconfig.py b/catalogbuilder/intakebuilder/builderconfig.py similarity index 100% rename from intakebuilder/builderconfig.py rename to catalogbuilder/intakebuilder/builderconfig.py diff --git a/intakebuilder/catalogcols.py b/catalogbuilder/intakebuilder/catalogcols.py similarity index 100% rename from intakebuilder/catalogcols.py rename to catalogbuilder/intakebuilder/catalogcols.py diff --git a/intakebuilder/config.yaml b/catalogbuilder/intakebuilder/config.yaml similarity index 100% rename from intakebuilder/config.yaml rename to catalogbuilder/intakebuilder/config.yaml diff --git a/intakebuilder/configparser.py b/catalogbuilder/intakebuilder/configparser.py similarity index 100% rename from intakebuilder/configparser.py rename to catalogbuilder/intakebuilder/configparser.py diff --git a/intakebuilder/getinfo.py b/catalogbuilder/intakebuilder/getinfo.py similarity index 100% rename from intakebuilder/getinfo.py rename to catalogbuilder/intakebuilder/getinfo.py diff --git a/intakebuilder/gfdlcrawler.py b/catalogbuilder/intakebuilder/gfdlcrawler.py similarity index 100% rename from intakebuilder/gfdlcrawler.py rename to catalogbuilder/intakebuilder/gfdlcrawler.py diff --git a/intakebuilder/localcrawler.py b/catalogbuilder/intakebuilder/localcrawler.py similarity index 100% rename from intakebuilder/localcrawler.py rename to catalogbuilder/intakebuilder/localcrawler.py diff --git a/intakebuilder/s3crawler.py b/catalogbuilder/intakebuilder/s3crawler.py similarity index 100% rename from intakebuilder/s3crawler.py rename to catalogbuilder/intakebuilder/s3crawler.py diff --git a/intakebuilder/table.yaml b/catalogbuilder/intakebuilder/table.yaml similarity index 100% rename from intakebuilder/table.yaml rename to catalogbuilder/intakebuilder/table.yaml diff --git a/scripts/__init__.py b/catalogbuilder/scripts/__init__.py similarity index 100% rename from scripts/__init__.py rename to catalogbuilder/scripts/__init__.py diff --git a/scripts/configs/config-example.yml b/catalogbuilder/scripts/configs/config-example.yml similarity index 100% rename from scripts/configs/config-example.yml rename to catalogbuilder/scripts/configs/config-example.yml diff --git a/scripts/configs/config-template.yaml b/catalogbuilder/scripts/configs/config-template.yaml similarity index 100% rename from scripts/configs/config-template.yaml rename to catalogbuilder/scripts/configs/config-template.yaml diff --git a/scripts/gen_intake_gfdl.py b/catalogbuilder/scripts/gen_intake_gfdl.py similarity index 100% rename from scripts/gen_intake_gfdl.py rename to catalogbuilder/scripts/gen_intake_gfdl.py diff --git a/scripts/gen_intake_gfdl_notebook.ipynb b/catalogbuilder/scripts/gen_intake_gfdl_notebook.ipynb similarity index 99% rename from scripts/gen_intake_gfdl_notebook.ipynb rename to catalogbuilder/scripts/gen_intake_gfdl_notebook.ipynb index 5ec2ff2..ef4115d 100644 --- a/scripts/gen_intake_gfdl_notebook.ipynb +++ b/catalogbuilder/scripts/gen_intake_gfdl_notebook.ipynb @@ -56,7 +56,7 @@ } ], "source": [ - "from scripts import gen_intake_gfdl\n", + "from catalogbuilder.scripts import gen_intake_gfdl\n", "import sys,os\n", "\n", "######USER input begins########\n", diff --git a/scripts/gen_intake_gfdl_runner.py b/catalogbuilder/scripts/gen_intake_gfdl_runner.py similarity index 67% rename from scripts/gen_intake_gfdl_runner.py rename to catalogbuilder/scripts/gen_intake_gfdl_runner.py index 920ede8..2f487b9 100755 --- a/scripts/gen_intake_gfdl_runner.py +++ b/catalogbuilder/scripts/gen_intake_gfdl_runner.py @@ -1,6 +1,7 @@ #!/usr/bin/env python -from scripts import gen_intake_gfdl +#TODO test after conda pkg is published and make changes as needed +from catalogbuilder.scripts import gen_intake_gfdl import sys input_path = "/archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp/" diff --git a/scripts/gen_intake_gfdl_runner_config.py b/catalogbuilder/scripts/gen_intake_gfdl_runner_config.py similarity index 78% rename from scripts/gen_intake_gfdl_runner_config.py rename to catalogbuilder/scripts/gen_intake_gfdl_runner_config.py index c7e019f..86fb91f 100755 --- a/scripts/gen_intake_gfdl_runner_config.py +++ b/catalogbuilder/scripts/gen_intake_gfdl_runner_config.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from scripts import gen_intake_gfdl +from catalogbuilder.scripts import gen_intake_gfdl import sys sys.argv = ['input_path','--config', '/home/a1r/github/CatalogBuilder/scripts/configs/config-example.yml'] diff --git a/scripts/gen_intake_local.py b/catalogbuilder/scripts/gen_intake_local.py similarity index 100% rename from scripts/gen_intake_local.py rename to catalogbuilder/scripts/gen_intake_local.py diff --git a/scripts/gen_intake_s3.py b/catalogbuilder/scripts/gen_intake_s3.py similarity index 100% rename from scripts/gen_intake_s3.py rename to catalogbuilder/scripts/gen_intake_s3.py diff --git a/scripts/test_catalog.py b/catalogbuilder/scripts/test_catalog.py similarity index 97% rename from scripts/test_catalog.py rename to catalogbuilder/scripts/test_catalog.py index c52e8b6..ed44d95 100755 --- a/scripts/test_catalog.py +++ b/catalogbuilder/scripts/test_catalog.py @@ -23,7 +23,7 @@ def main(json_path,json_template_path,test_failure): if json_template_path: json_template = json.load(open(json_template_path)) else: - json_template = json.load(open('cats/gfdl_template.json')) + json_template = json.load(open('catalogbuilder/cats/gfdl_template.json')) #Validate JSON against JSON template comp = (diff(j,json_template)) diff --git a/meta.yaml b/meta.yaml index 2fad1d6..87791fa 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,9 +1,10 @@ package: - name: intakebuilder + name: catalogbuilder version: 2.0.1 source: - git_url: https://github.com/aradhakrishnanGFDL/CatalogBuilder.git + git_url: https://github.com/NOAA-GFDL/CatalogBuilder.git + git_branch: subpkgtest path: . build: @@ -16,16 +17,17 @@ requirements: - pip run: - python + - pytest - click - - pandas - xarray - - pytest - - conda-forge::intake-esm=2023.7.7 + - pandas - jsondiff + - conda-forge::intake-esm=2023.7.7 + - boto3 test: imports: - - intakebuilder - - scripts.gen_intake_gfdl - + - catalogbuilder + - catalogbuilder.scripts + - catalogbuilder.cats about: home: diff --git a/setup.py b/setup.py index 3231e20..110a25d 100644 --- a/setup.py +++ b/setup.py @@ -14,8 +14,9 @@ 'click', 'xarray', 'pandas', - 'boto3', - 'botocore' + 'jsondiff', + 'intake-esm', + 'boto3' ] ) diff --git a/tests/test_config.yaml b/tests/test_config.yaml index 03f7806..803f1b8 100644 --- a/tests/test_config.yaml +++ b/tests/test_config.yaml @@ -38,4 +38,4 @@ output_file_template: ['modeling_realm','temporal_subset','variable_id'] ####################################################### input_path: "archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp/" #"ENTER INPUT PATH HERE" #Example: /Users/ar46/archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp/" -output_path: "cats/gfdl_autotest_from_yaml" # ENTER NAME OF THE CSV AND JSON, THE SUFFIX ALONE. e.g catalog (the builder then generates catalog.csv and catalog.json. This can also be an absolute path) +output_path: "catalogbuilder/cats/gfdl_autotest_from_yaml" # ENTER NAME OF THE CSV AND JSON, THE SUFFIX ALONE. e.g catalog (the builder then generates catalog.csv and catalog.json. This can also be an absolute path) diff --git a/tests/test_import.py b/tests/test_import.py index 794b49a..e5d5213 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -1,10 +1,21 @@ +import os,sys def check_import(): try: - from intakebuilder import getinfo, localcrawler, CSVwriter - print("Imported intakebuilder and getinfo, localcrawler, CSVwriter ") + from intakebuilder import gfdlcrawler, CSVwriter, configparser except ImportError: - raise ImportError('Error importing intakebuilder and other packages') - return -97 + print("The module intakebuilder is not installed. Do you have intakebuilder in your sys.path or have you activated the conda environment with the intakebuilder package in it? ") + print("Attempting again with adjusted sys.path ") + try: + #candobetter with actual conda package built before this, but this is a decent test for those that do not use conda package + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+"/catalogbuilder") + from intakebuilder import gfdlcrawler, CSVwriter, builderconfig, configparser + except: + print("Unable to adjust sys.path") + raise ImportError('Error importing intakebuilder and other packages') + return -97 return True + def test_import(): assert check_import() == True + + diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py index 38f6b41..13b6aec 100755 --- a/tests/test_ingestion.py +++ b/tests/test_ingestion.py @@ -21,7 +21,7 @@ def load_cat(catspec=None): return cat def test_loadcat(): #TODO generate csv on the fly, check if its readable etc - catspec = pathlib.Path(os.path.dirname(__file__)).parent / 'cats/gfdl_template.json' + catspec = pathlib.Path(os.path.dirname(__file__)).parent / 'catalogbuilder/cats/gfdl_template.json' #TODO generate test catalog on the fly, push spec to the test directory cat = load_cat((str(catspec))) try: