Merge pull request NASA-Openscapes#306 from eeholmes/main

Containerized environment(s) for earthdata cloud cookbook

.devcontainer/Dockerfile

@@ -0,0 +1,64 @@
+## devcontainer-focused Rocker
+FROM ghcr.io/rocker-org/devcontainer/tidyverse:4.3
+
+## latest version of geospatial libs
+RUN /rocker_scripts/experimental/install_dev_osgeo.sh
+
+## install vim for convenience
+## texlive installed for install_jupyter.sh because it tries to install some basic tex libraries
+RUN apt-get update -qq && apt-get -y install vim texlive
+
+# standard python/jupyter setup
+ENV NB_USER=rstudio
+ENV VIRTUAL_ENV=/opt/venv
+ENV PATH=${VIRTUAL_ENV}/bin:${PATH}
+RUN wget https://github.com/rocker-org/rocker-versioned2/raw/master/scripts/install_jupyter.sh && \
+  bash -e install_jupyter.sh && \
+  rm install_jupyter.sh && \
+  chown ${NB_USER}:staff -R ${VIRTUAL_ENV}
+
+# Set up conda
+ENV CONDA_ENV=/opt/miniforge3
+ENV PATH=${PATH}:$CONDA_ENV/bin
+RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \
+    bash Miniforge3-$(uname)-$(uname -m).sh -b -p ${CONDA_ENV} && \
+    chown ${NB_USER}:staff -R ${CONDA_ENV}
+
+# podman doesn't not understand group permissions
+# RUN chown ${NB_USER}:staff -R ${R_HOME}/site-library
+
+# some teaching preferences; might replace with gh-creds
+RUN git config --system pull.rebase false && \
+    git config --system credential.helper 'cache --timeout=36000'
+
+## codeserver; Add VSCode button
+RUN curl -fsSL https://code-server.dev/install.sh | sh && rm -rf .cache
+
+## Install the R packages into site library
+COPY install.R install.R
+RUN Rscript install.R && rm install.R
+
+## Openscapes-specific configs
+USER rstudio
+WORKDIR /home/rstudio
+# make bash the user default
+RUN usermod -s /bin/bash rstudio
+
+# install into the default venv environment
+COPY nasa-requirements.txt requirements.txt
+RUN python -m pip install -r requirements.txt && rm requirements.txt
+
+# Create a conda-based env and install into it without using conda init/conda activate
+# (this yaml file doesn't include everything from pangeo, consider a different one...)
+# pangeo uses the name 'notebook'
+ENV ENV_NAME=notebook
+ENV MY_ENV=${CONDA_ENV}/envs/${ENV_NAME}
+RUN wget https://github.com/NASA-Openscapes/corn/raw/main/ci/environment.yml && \
+    conda env create -p ${MY_ENV} -f environment.yml
+
+# This won't be the default environment but we register it
+RUN ${MY_ENV}/bin/python -m pip install ipykernel && \
+    ${MY_ENV}/bin/python -m ipykernel install --prefix /opt/venv --name=${ENV_NAME}
+
+
+

.devcontainer/conda.Dockerfile

@@ -0,0 +1,59 @@
+# devcontainer-focused Rocker
+FROM ghcr.io/rocker-org/devcontainer/tidyverse:4.3
+
+## Set the default shell
+ENV SHELL=/bin/bash
+
+# latest version of geospatial libs -- big! 
+# RUN /rocker_scripts/experimental/install_dev_osgeo.sh && rm -rf /build_*
+# install vim for convenience
+
+RUN apt-get update -qq && apt-get -y install vim
+
+# some teaching preferences
+RUN git config --system pull.rebase false && \
+    git config --system credential.helper 'cache --timeout=36000'
+
+# codeserver; Adds the VSCode button
+RUN curl -fsSL https://code-server.dev/install.sh | sh && rm -rf .cache
+
+# Rename the rstudio user as jovyan
+RUN usermod -l jovyan rstudio && \
+    usermod -d /home/jovyan -m jovyan && \
+    groupmod -n jovyan rstudio
+
+
+# Set up conda
+ENV NB_USER=jovyan
+ENV CONDA_DIR=/opt/miniforge3
+ENV CONDA_ENV=openscapes
+ENV NB_PYTHON_PREFIX=${CONDA_DIR}/envs/${CONDA_ENV} 
+ENV PATH=${NB_PYTHON_PREFIX}/bin:/${CONDA_DIR}/bin:${PATH}
+
+# install conda
+RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \
+    bash Miniforge3-$(uname)-$(uname -m).sh -b -p ${CONDA_DIR} && \
+    chown ${NB_USER}:staff -R ${CONDA_DIR} && \
+    rm -f Miniforge3*.sh *.deb
+
+# conda decorators
+RUN conda init --system
+RUN echo ". ${CONDA_DIR}/etc/profile.d/conda.sh ; conda activate ${CONDA_ENV}" > /etc/profile.d/init_conda.sh
+
+# Tell RStudio how to find Python
+RUN echo "PATH=${PATH}" >>"${R_HOME}/etc/Renviron.site"
+
+# install R packages into the site library
+COPY install.R install.R
+RUN Rscript install.R && rm install.R
+
+# Standard user setup here
+USER ${NB_USER} 
+WORKDIR /home/${NB_USER}
+# make bash default shell
+RUN usermod -s /bin/bash ${NB_USER} 
+
+COPY environment.yml environment.yml
+RUN conda env update -f environment.yml && conda clean --all
+RUN rm environment.yml
+

.devcontainer/conda2.Dockerfile

@@ -0,0 +1,47 @@
+# devcontainer-focused Rocker
+FROM ghcr.io/rocker-org/devcontainer/tidyverse:4.3
+
+# latest version of geospatial libs
+RUN /rocker_scripts/experimental/install_dev_osgeo.sh && rm -rf /build_*
+RUN apt-get update -qq && apt-get -y install vim
+
+# podman doesn't understand group permissions
+RUN chown rstudio:staff ${R_HOME}/site-library
+
+# some teaching preferences
+RUN git config --system pull.rebase false && \
+    git config --system credential.helper 'cache --timeout=36000'
+
+# codeserver
+RUN curl -fsSL https://code-server.dev/install.sh | sh
+
+# Set up conda
+ENV NB_USER=rstudio
+ENV CONDA_ENV=/opt/miniforge3
+ENV PATH=${CONDA_ENV}/bin:${PATH}
+RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \
+    bash Miniforge3-$(uname)-$(uname -m).sh -b -p ${CONDA_ENV} && \
+    chown ${NB_USER}:staff -R ${CONDA_ENV} && \
+    rm Miniforge3*.sh *.deb
+
+# Set up a primary conda environment distinct from (base)
+ENV MY_ENV=${CONDA_ENV}/envs/openscapes
+ENV PATH=${MY_ENV}/bin:${PATH}
+RUN echo "PATH=${PATH}" >>"${R_HOME}/etc/Renviron.site"
+
+# Initialize conda by default for all users:
+# RUN conda init --system # will cause terminal to start in base 
+
+# Standard user setup here
+USER ${NB_USER} 
+WORKDIR /home/${NB_USER}
+RUN usermod -s /bin/bash ${NB_USER} 
+
+# install into the default environment
+COPY install.R install.R
+RUN Rscript install.R && rm install.R
+RUN rm install.R
+
+COPY environment.yml environment.yml
+RUN conda env create -p ${MY_ENV} -f environment.yml && conda clean --all
+

.devcontainer/devcontainer.json

@@ -0,0 +1,44 @@
+{
+	  "image": "ghcr.io/eeholmes/earthdata-cloud-cookbook/cookbook:venv",
+    // container run args for gdal netcdf vsi access
+	  "runArgs": [ "--security-opt", "seccomp=unconfined" ],
+	  "name": "NASA TOPS Environment",
+	  "waitFor": "onCreateCommand",
+	  "features": {
+		"ghcr.io/rocker-org/devcontainer-features/quarto-cli:1": {}
+	  },
+	  "customizations": {
+		"codespaces": {
+			"openFiles": ["README.md"]
+		  },
+		  "vscode": {
+		  "settings": {
+			"r.rterm.linux": "/usr/local/bin/radian",
+			"r.bracketedPaste": true,
+			"r.plot.useHttpgd": true,
+			"[r]": {
+			  "editor.wordSeparators": "`~!@#%$^&*()-=+[{]}\\|;:'\",<>/?"
+			}
+		  },
+		  "extensions": [
+			"reditorsupport.r",
+			"rdebugger.r-debugger",
+			"ms-toolsai.jupyter",
+			"ms-python.python"			
+			]
+		}
+	  },
+	  // Forward the RStudio ports
+	  "forwardPorts": [8787],
+	  "portsAttributes": {
+		"8787": {
+			"label": "Rstudio",
+			"requireLocalPort": true,
+			"onAutoForward": "ignore"
+		}
+	  },
+	  // Use 'postCreateCommand' to run commands after the container is created.
+	  "postCreateCommand": "bash .devcontainer/setup.sh",
+	  "postAttachCommand": "sudo rstudio-server start &> /dev/null && bash .devcontainer/welcome.sh",
+	  "remoteUser": "rstudio"
+	}

.devcontainer/environment.yml

@@ -0,0 +1,88 @@
+name: openscapes
+channels:
+  - conda-forge
+dependencies:
+  - python=3.10
+  - dask
+  - dask-gateway
+  - graphviz
+  - distributed
+  - rioxarray
+  - xarray>=2022.12
+  - h5netcdf
+  - netcdf4>=1.6.4
+  - h5py
+  - gdal~=3.8
+  - geoviews
+  - holoviews=1.16.2
+  - matplotlib-base
+  - seaborn
+  - intake
+  - hvplot
+  - datashader
+  - pyproj
+  - bqplot
+  - geopandas
+  - zarr
+  - cartopy
+  - pynco
+  - rasterio
+  - shapely
+  - pyresample
+  - icepyx
+  - joblib
+  - pystac-client
+  - odc-stac
+  - pydap
+  - lxml
+  - ipyleaflet
+  # when jlab 4 gets released we can upgrade dask-labextension
+  # - dask-labextension
+  # - dask-labextension==6.2.0
+  - dask-labextension
+  - jupyter-server-proxy
+  - jupyter-vscode-proxy
+  - jupyter-rsession-proxy
+  - ipywidgets
+  - jupyter-book
+  - jupyterlab>4
+  - jupyterlab-myst
+  - jupyterhub-singleuser
+  - jupyterlab-geojson
+  - jupyterlab-favorites
+  - jupyterlab-git
+  - jupyter-resource-usage
+  - jupyterlab-h5web
+  - hdf5plugin
+  - ipympl
+  - conda-lock
+  - pooch
+  - earthaccess>=0.8.2
+  - sliderule>=3.6.0
+  # QGIS
+  - qgis~=3.28
+  - pyopencl
+  - ocl-icd-system
+  - websockify>=0.10
+  # /QGIS 
+  - h5coro
+  - git >=2.39
+  - itslive
+  - rasterstats
+  - kerchunk
+  - rechunker
+  - pqdm
+  - spectral
+  - scikit-image
+  - coiled
+  - s3fs
+  - nbgitpuller
+  - awscliv2
+  - jupyter-server-proxy
+  - jupyter-vscode-proxy
+  # https://github.com/jupyterlab/jupyter-collaboration/issues/202
+  # - jupyter-collaboration
+  - code-server >=3.2
+  - pip
+platforms:
+  - linux-64

.devcontainer/install.R

@@ -0,0 +1,14 @@
+#! /usr/local/bin/Rscript
+# install R dependencies
+
+# We could use renv.lock approach here instead, but will force re-creation of environment from scratch
+# Does not provide a good way to ensure that sf/terra/gdalcubes are installed from source while other packages can be binary
+# Likewise, pak insists on installing old gdal from apt instead of respecting system library source builds
+install.packages("pak")
+pak::pkg_install(c("rstac", "spData", "earthdatalogin", "quarto", "aws.s3", "tmap", "reticulate"))
+pak::pkg_install('r-tmap/tmap')
+
+#pak::pkg_install("httpgd")
+#pak::pkg_install(c("IRkernel", "languageserver"))
+#IRkernel::installspec()
+

.devcontainer/jupyter-environment.yml

@@ -0,0 +1,21 @@
+name: openscapes
+channels:
+  - conda-forge
+dependencies:
+  - python=3.10
+  - dask-labextension
+  - jupyter-server-proxy
+  - jupyter-vscode-proxy
+  - jupyter-rsession-proxy
+  - ipywidgets
+  - jupyter-book
+  - jupyterlab>4
+  - jupyterlab-myst
+  - jupyterhub-singleuser
+  - jupyterlab-geojson
+  - jupyterlab-favorites
+  - jupyterlab-git
+  - jupyter-resource-usage
+  - jupyterlab-h5web
+  - code-server >=3.2
+  - pip

.devcontainer/nasa-requirements.txt

@@ -0,0 +1,41 @@
+jupyterhub
+jupyterlab
+notebook
+jupyterlab-quarto
+jupyter-rsession-proxy
+jupyter-vscode-proxy
+jupyter-console
+jupyter-events
+jupyter-lsp
+jupyter_client
+jupyter_core
+jupyter_server
+jupyter_server_terminals
+jupyterlab-pygments
+jupyterlab-widgets
+jupyterlab_server
+gh-scoped-creds
+nbconvert
+nbformat
+odc-stac
+planetary-computer
+polars
+pyarrow
+pystac-client
+scikit-image
+hvplot
+geoviews
+earthaccess
+pyresample
+contextily
+rioxarray
+geopandas
+rasterstats
+dask
+distributed
+h5netcdf
+h5py
+netCDF4
+zarr
+lxml
+