Merge branch 'release/3.1.x' into fix/xd_support_interactive

This merges the release/3.1.x branch into the fix/xd_support_interactive
branch, and consolidates some significant changes to the wavelength
calibration interactive code.

.gitignore

@@ -157,3 +157,6 @@ dmypy.json
 **/.adcc
 **/dragons_test_inputs
 **/spec_data.json
+
+# VSCode settings
+.vscode/settings.json

.readthedocs.yaml

@@ -27,4 +27,4 @@ sphinx:
 python:
    install:
    - requirements: requirements.txt
-#   - requirements: docs/requirements.txt
+   - requirements: doc/requirements.txt

Jenkinsfile

@@ -35,11 +35,13 @@ pipeline {
         buildDiscarder(logRotator(numToKeepStr: '5'))
         timestamps()
         timeout(time: 6, unit: 'HOURS')
+        timeout(time: 6, unit: 'HOURS')
     }
 
     environment {
         MPLBACKEND = "agg"
         PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
+        PATH = "$JENKINS_CONDA_HOME/bin:$PATH"
     }
 
     stages {
@@ -51,6 +53,31 @@ pipeline {
             }
         }
 
+        stage('Pre-install') {
+            agent { label "conda" }
+            environment {
+                TMPDIR = "${env.WORKSPACE}/.tmp/conda/"
+            }
+            steps {
+                echo "Update the Conda base install for all on-line nodes"
+                checkout scm
+                sh '.jenkins/scripts/setup_agent.sh'
+                echo "Create a trial Python 3.10 env, to cache new packages"
+                sh 'tox -e py310-noop -v -r -- --basetemp=${DRAGONS_TEST_OUT} ${TOX_ARGS}'
+            }
+            post {
+                always {
+                    echo "Deleting conda temp workspace ${env.WORKSPACE}"
+                    cleanWs()
+                    dir("${env.WORKSPACE}@tmp") {
+                      deleteDir()
+                    }
+                }
+                echo "Step would notify STARTED when dragons_ci is available"
+                // sendNotifications 'STARTED'
+            }
+        }
+
         stage('Pre-install') {
             agent { label "conda" }
             environment {
@@ -92,6 +119,7 @@ pipeline {
                         echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                         checkout scm
                         sh '.jenkins/scripts/setup_dirs.sh'
+                        sh '.jenkins/scripts/setup_dirs.sh'
                         echo "Running tests with Python 3.10"
                         sh 'tox -e py310-unit -v -r -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/unittests_results.xml ${TOX_ARGS}'
                         echo "Reportint coverage to CodeCov"
@@ -130,6 +158,7 @@ pipeline {
                         checkout scm
                         echo "${env.PATH}"
                         sh '.jenkins/scripts/setup_dirs.sh'
+                        sh '.jenkins/scripts/setup_dirs.sh'
                         echo "Integration tests"
                         sh 'tox -e py310-integ -v -r -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/integration_results.xml ${TOX_ARGS}'
                         echo "Reporting coverage"
@@ -163,6 +192,7 @@ pipeline {
                         checkout scm
                         echo "${env.PATH}"
                         sh '.jenkins/scripts/setup_dirs.sh'
+                        sh '.jenkins/scripts/setup_dirs.sh'
                         echo "Regression tests"
                         sh 'tox -e py310-reg -v -r -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/regression_results.xml ${TOX_ARGS}'
                         echo "Reporting coverage"
@@ -389,6 +419,7 @@ pipeline {
                         echo "Running build #${env.BUILD_ID} on ${env.NODE_NAME}"
                         checkout scm
                         sh '.jenkins/scripts/setup_dirs.sh'
+                        sh '.jenkins/scripts/setup_dirs.sh'
                         echo "Running tests"
                         sh 'tox -e py310-gmosls -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/gmosls_results.xml ${TOX_ARGS}'
                         echo "Reporting coverage"
@@ -427,6 +458,7 @@ pipeline {
                         checkout scm
                         echo "${env.PATH}"
                         sh '.jenkins/scripts/setup_dirs.sh'
+                        sh '.jenkins/scripts/setup_dirs.sh'
                         echo "Slow tests"
                         sh 'tox -e py310-slow -v -- --basetemp=${DRAGONS_TEST_OUT} --junit-xml reports/slow_results.xml ${TOX_ARGS}'
                         echo "Reporting coverage"
@@ -454,11 +486,15 @@ pipeline {
         success {
             echo "Step would notify SUCCESSFUL when dragons_ci is available"
             // sendNotifications 'SUCCESSFUL'
+            echo "Step would notify SUCCESSFUL when dragons_ci is available"
+            // sendNotifications 'SUCCESSFUL'
 //            deleteDir() /* clean up our workspace */
         }
         failure {
             echo "Step would notify FAILED when dragons_ci is available"
             // sendNotifications 'FAILED'
+            echo "Step would notify FAILED when dragons_ci is available"
+            // sendNotifications 'FAILED'
 //            deleteDir() /* clean up our workspace */
         }
         always {

astrodata/doc/usermanual/data.rst

@@ -674,7 +674,7 @@ add a variance plane to our raw data frame.
     >>> # Here we work on the NDAstroData object to have the variance
     >>> # trimmed automatically to the same size as the science array.
     >>> # To reassign the cropped NDAstroData, we use the reset() method.
-    >>> ad[0].reset(ad[0].nddata[datasec.y1:datasec.y2, datasec.x1:datasec.x2]
+    >>> ad[0].reset(ad[0].nddata[datasec.y1:datasec.y2, datasec.x1:datasec.x2])
 
     >>> # Now look at the dimensions of the first extension, science
     >>> # and variance.  That extension is smaller than the others.
@@ -771,9 +771,9 @@ attention.
     >>> intensity = adcube[0].data[:, 24, 13]   # (wlen, y, x)
 
     # We plot
-    plt.clf()
-    plt.plot(wavelengths, intensity)
-    plt.show()
+    >>> plt.clf()
+    >>> plt.plot(wavelengths, intensity)
+    >>> plt.show()
 
 
 Plot Data
@@ -874,7 +874,7 @@ what ``imexam`` has to offer.
     >>> ds9.scale('zscale')
 
     # Run in interactive mode.  Try the various commands.
-   >>> ds9.imexam()
+    >>> ds9.imexam()
 
     # Use the programmatic interface
     # First initialize an Imexamine object.

doc/DRAGONS/changes.rst

@@ -27,6 +27,13 @@ code from "provenance history" to simply "history".
 * Whitespace now allowed in directory paths (if quoted), e.g.,
   ``databases = "~/.my dragons/dragons.db"``
 
+Bug Fixes
+---------
+
+**geminidr.gmos**
+
+* Fix the QE model selection for the GMOS-S EEV CDDs.
+
 
 3.1.0
 =====
@@ -39,9 +46,10 @@ the user need to be aware of.  Please read on.
 New Features
 ------------
 
-Science quality support for GMOS longslit spectroscopy.
+Science quality support for GMOS longslit spectroscopy, including nod-and-shuffle.
   Please refer to the tutorial, |GMOSLSTut|.  DRAGONS is now the official
-  software for reducing GMOS longslit data.
+  software for reducing GMOS longslit data in normal and nod-and-shuffle
+  mode.
 
 New browser-base interactive tools to support spectroscopy.
   The following primitives have an interactive mode that can be activated with

doc/DRAGONS/dragonsteam.rst

@@ -0,0 +1,36 @@
+.. dragonsteam.rst
+
+.. _dragonsteam:
+
+****************
+The DRAGONS Team
+****************
+
+Current DRAGONS Team
+====================
+(Alphabetical order)
+
+* Daniel Berke
+* Paul Hirst
+* David Jones
+* Kathleen Labrie
+* Chris Simpson
+* Olesja Smirnova
+* Monika Soraisam
+* James Turner
+* Bill Vacca
+
+Past DRAGONS Team Members
+=========================
+(Alphabetical order)
+
+* Ken Anderson
+* Ricardo Cárdenes
+* Simon Conseil
+* Oliver Oberdorf
+* Bruno Quint
+* Joanna Thomas-Osip
+
+Special Thanks
+==============
+* Vinicius Placco

doc/DRAGONS/index.rst

@@ -64,6 +64,11 @@ Releases
 
    releasenotes.rst
    changes.rst
+   dragonsteam.rst
+
+==========
+
+
 
 .. Developer Notes
 .. ---------------

doc/DRAGONS/releasenotes.rst

@@ -14,7 +14,7 @@ and new tutorials.  See the :ref:`changes` for all the details.
 
 This update of DRAGONS has four big changes over V3.0:
 
-* New science quality reduction for GMOS longslit data
+* New science quality reduction for GMOS longslit data, including nod-and-shuffle.
 * The alignment and stacking of GSAOI data is now done within DRAGONS and the]
   use of ``disco_stu`` is no longer required.
 * The calibration service has gone through a major refactoring.  Of direct
@@ -31,7 +31,7 @@ Science Quality reduction
    * NIRI imager
    * GSAOI imager
    * F2 imager
-   * GMOS longslit spectrograph
+   * GMOS longslit spectrograph (including nod-and-shuffle)
 
 For imaging and GMOS longslit spectroscopy, this software should be used
 instead of the Gemini IRAF package.

doc/requirements.txt

@@ -0,0 +1,2 @@
+# requirements only used for building documentation
+sphinx-rtd-theme

geminidr/core/parameters_preprocess.py

@@ -99,7 +99,7 @@ class separateSkyConfig(config.Config):
     suffix = config.Field("Filename suffix", str, "_skySeparated", optional=True)
     ref_obj = config.Field("Manually-assigned object files", str, None, optional=True)
     ref_sky = config.Field("Manually-assigned sky files", str, None, optional=True)
-    frac_FOV = config.RangeField("Field of view scaling for coaddition", float, 0.9, min=0.5, max=1)
+    frac_FOV = config.RangeField("Field of view scaling for coaddition", float, 0.9, min=0.3, max=1)
 
 
 class makeSkyConfig(associateSkyConfig, separateSkyConfig):
@@ -127,6 +127,7 @@ class subtractSkyConfig(config.Config):
     offset_sky = config.Field("Apply offset to sky frame to match science frame?", bool, False)
     sky = config.ListField("Sky frame to subtract", (str, AstroData), None, optional=True, single=True)
     save_sky = config.Field("Save sky frame to disk?", bool, False)
+    debug_threshold = config.Field("Convergence threshold when scaling", float, 0.001)
 
 
 class skyCorrectConfig(parameters_stack.stackSkyFramesConfig, subtractSkyConfig):

geminidr/core/parameters_spect.py

@@ -523,7 +523,7 @@ class separateSkyConfig(parameters_preprocess.separateSkyConfig):
 
 class skyCorrectFromSlitConfig(config.core_1Dfitting_config):
     suffix = config.Field("Filename suffix", str, "_skyCorrected", optional=True)
-    regions = config.Field("Sample regions", str, None, optional=True)
+    regions = config.Field("Sample regions. (eg. 100:150,251:264)", str, None, optional=True)
     aperture_growth = config.RangeField("Aperture avoidance distance (pixels)", float, 2, min=0)
     debug_plot = config.Field("Show diagnostic plots?", bool, False)
     interactive = config.Field("Run primitive interactively?", bool, False)

geminidr/core/primitives_calibdb.py

@@ -161,7 +161,15 @@ def _markAsCalibration(self, adinputs=None, suffix=None, update_datalab=True,
                 proc_suffix = f"_{mode}"
 
             if suffix:
-                proc_suffix += suffix
+                # the update_filename method, keeps the procmode suffix
+                # when it strips, if we don't remove it here, it will duplicate
+                # it,  eg _ql_ql_flat.  This can happen when storing after the
+                # fact, eg. the file is already _ql_flat.  Here, if the
+                # proc_suffix is already in the filename use only suffix.
+                if proc_suffix in ad.filename:
+                    proc_suffix = suffix
+                else:
+                    proc_suffix += suffix
                 strip = True
             else:
                 strip = False

geminidr/core/primitives_preprocess.py

@@ -33,6 +33,8 @@
 from recipe_system.utils.decorators import parameter_override, capture_provenance
 from recipe_system.utils.md5 import md5sum
 
+from gempy.utils.errors import ConvergenceError
+
 from . import parameters_preprocess
 
 
@@ -1648,6 +1650,7 @@ def subtractSky(self, adinputs=None, **params):
         reset_sky = params["reset_sky"]
         scale = params["scale_sky"]
         zero = params["offset_sky"]
+        debug_threshold = params["debug_threshold"]
         if scale and zero:
             log.warning("Both the scale_sky and offset_sky parameters are set. "
                         "Setting offset_sky=False.")
@@ -1657,7 +1660,7 @@ def subtractSky(self, adinputs=None, **params):
         # in gt.measure_bg_from_image()
         sampling = 1 if adinputs[0].instrument() == 'GNIRS' else 10
         skyfunc = partial(gt.measure_bg_from_image, value_only=True,
-                          sampling=sampling)
+                          sampling=sampling, gaussfit=True)
 
         for ad, ad_sky in zip(*gt.make_lists(adinputs, params["sky"],
                                              force_ad=True)):
@@ -1674,8 +1677,26 @@ def subtractSky(self, adinputs=None, **params):
                             f"the science frame {ad.filename}")
                 if scale or zero:
                     # This actually does the sky subtraction as well
-                    factors = [gt.sky_factor(ext, ext_sky, skyfunc, multiplicative=scale)
-                               for ext, ext_sky in zip(ad, ad_sky)]
+                    try:
+                        factors = [gt.sky_factor(ext, ext_sky, skyfunc,
+                                                 multiplicative=scale,
+                                                 threshold=debug_threshold)
+                                   for ext, ext_sky in zip(ad, ad_sky)]
+                    except ConvergenceError as error:
+                        log.warning(f"The scaling of sky using a gaussian fit "
+                                    f"did not converge.  \n"
+                                    f"Using the median method instead.")
+                        skyfunc = partial(gt.measure_bg_from_image,
+                                          value_only=True,
+                                          sampling=sampling, gaussfit=False)
+                        try:
+                            factors = [gt.sky_factor(ext, ext_sky, skyfunc,
+                                                     multiplicative=scale)
+                                       for ext, ext_sky in zip(ad, ad_sky)]
+                        except ConvergenceError as error:
+                            log.error(f"Failed to scaled sky.")
+                            raise(error)
+
                     for ext_sky, factor in zip(ad_sky, factors):
                         log.fullinfo("Applying {} of {} to extension {}".
                                 format("scaling" if scale else "offset",

geminidr/doc/tutorials/F2Img-DRTutorial/index.rst

@@ -10,6 +10,7 @@ DRAGONS - Flamingos 2 Data Reduction Tutorial
 This is a brief tutorial on how to reduce Flamingos-2 images using DRAGONS (Data
 Reduction for Astronomy from Gemini Observatory North and South).
 
+[Return to the main |DRAGONS| page.]
 
 .. toctree::
    :maxdepth: 2

geminidr/doc/tutorials/GMOSImg-DRTutorial/ex2_gmosim_separateCCDs_cmdline.rst

@@ -337,6 +337,11 @@ appear on two CCDs.  This is because the each CCD has been stacked individually
 and because of the dithers some sources ended up moving from to the adjacent
 CCD.
 
+.. note::
+
+    ``ds9`` must be launched by the user ahead of running the display primitive.
+    (``ds9&`` on the terminal prompt.)
+
 The output stack units are in electrons (header keyword BUNIT=electrons).
 The output stack is stored in a multi-extension FITS (MEF) file.  The science
 signal is in the "SCI" extension, the variance is in the "VAR" extension, and

geminidr/doc/tutorials/GMOSImg-DRTutorial/index.rst

@@ -12,6 +12,8 @@ based on information found in the `GEMINI GMOS
 WebPage <https://www.gemini.edu/sciops/instruments/gmos/>`_ and in the
 |DRAGONS| Documentation
 
+[Return to the main |DRAGONS| page.]
+
 .. toctree::
    :maxdepth: 2
    :numbered:

geminidr/doc/tutorials/GMOSLS-DRTutorial/01_overview.rst

@@ -16,7 +16,7 @@ reduction of GMOS imaging data, see |GMOSImgTut|.
 
 Here is a tutorial that you, the reader, can run and experiment with.  This
 document comes with a downloadable data package that contains all the data
-you need to run the example presented.  Instructions on where to get that
+you need to run the examples presented.  Instructions on where to get that
 package and how to set things up are given in :ref:`datasetup`.
 
 The GMOS longslit tutorial series contains several scientific examples

geminidr/doc/tutorials/GMOSLS-DRTutorial/conf.py

@@ -151,6 +151,7 @@ def setup(app):
 .. |astrodatauser| replace:: :raw-html:`<a href="https://astrodata.readthedocs.io/en/{v}/usermanual/index.html" target="_blank">Astrodata User Manual</a>`
 
 .. |GMOSImgTut| replace:: :raw-html:`<a href="http://GMOSImg-DRTutorial.readthedocs.io/en/{v}">GMOS Imaging Data Reduction Tutorial</a>`
+.. |DRAGONS| replace:: :raw-html:`<a href="https://dragons.readthedocs.io/en/{v}/" target="_blank">DRAGONS</a>`
 
 .. |RSUserInstall| replace:: :raw-html:`<a href="https://dragons.readthedocs.io/projects/recipe-system-users-manual/en/{v}/install.html" target="_blank">DRAGONS Installation Instructions</a>`
 """.format(v = rtdurl)