docker implementation

docker/README.md

@@ -0,0 +1,2 @@
+# Docker
+These dockerfiles contain runtime environments necessary for using the perl and python versions of FUNC-E. To use, run them with a bash entrypoint (which is done already in the kube deployment section), then exec into them to run experiments.

docker/pl.Dockerfile

@@ -0,0 +1,15 @@
+### Image for FUNC-E perl
+FROM rocker/rstudio
+LABEL maintainer="Reed Bender <[email protected]>"
+
+# Download dependencies for FUNC_E
+RUN apt-get update && \
+cpan App::cpanminus && \
+cpan inc::latest && \
+cpan IPC::Run \
+cpan Getopt::Long && \
+cpan Text::NSP::Measures::2D::Fisher::right && \
+cpan List::Util && \
+cpan Math::Complex && \
+cpan Math::BigFloat && \
+cpan Statistics::R 

docker/py.Dockerfile

@@ -0,0 +1,13 @@
+FROM ubuntu:18.04
+
+# install system dependencies
+RUN apt-get update -qq && apt-get install -qq -y curl git python3-dev python3-pip && ln -s /usr/bin/python3 python
+
+#changing working directory in Docker container
+WORKDIR /app
+
+# copy data from local into Docker container
+ADD . /app/
+
+# install python dependencies
+RUN pip3 install -r requirements.txt

kube/README.md

@@ -0,0 +1,11 @@
+# Deploying to Kubernetes
+
+The important thing about deployment is that you can either deploy the perl version (which has more features) or the python version (which is probably slightly more easily understood). If you'd like the perl version, edit the container image field in the deploy to `mrbende/funce-image:01`, and for python to `ebensma/pyfunce:latest`.
+
+## Deployment Instructions
+
+* Launch the deployment with `kubectl apply -f kube/deploy.yaml`
+* Load the data you'd like to use on Kubernetes into the PVC using `./kube/kube-load.sh <pvc> <folder-to-copy>`
+* Get a shell into the pod you've created with `kubectl exec -it <pod name> -- /bin/bash`
+* If you're using the python version, you'll launch into the `/app` folder, which contains some demo materials. To find what you've copied into the PVC, navigate to the `/workspaces/your-username` folder
+* Run whatever FUNC-E tests you'd like, then exit the pod and save the results to your local machine using `./kube/kube-save.sh <pvc> <pvc-path-to-results>` .

kube/deploy.yaml

@@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: func-e-k8s
+  labels:
+    app: func-e-k8s
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: func-e-k8s
+  template:
+    metadata:
+      labels:
+        app: func-e-k8s
+    spec:
+      containers:               
+      - name: func-e-k8s
+        image: ebensma/pyfunce:latest
+        command: [ "/bin/bash", "-c", "--" ]
+        args: [ "tail -f /dev/null" ]
+        resources:
+          requests:
+            cpu: 1
+            memory: 64Gi
+          limits:
+            cpu: 1
+            memory: 64Gi
+        volumeMounts:
+        - name: vol-1
+          mountPath: /workspace
+      restartPolicy: Always
+      volumes:
+        - name: vol-1
+          persistentVolumeClaim:
+            claimName: deepgtex-prp # Enter valid PVC

kube/kube-load.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Load input data to a Persistent Volume on a Kubernetes cluster.
+
+# parse command-line arguments
+if [[ $# != 2 ]]; then
+	echo "usage: $0 <pvc-name> <local-path>"
+	exit -1
+fi
+
+PVC_NAME="$1"
+PVC_PATH="/workspace"
+POD_FILE="pod.yaml"
+POD_NAME="$USER-load-$(printf %04x $RANDOM)"
+LOCAL_PATH="$(realpath $2)"
+
+# create pod config file
+cat > $POD_FILE <<EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  name: $POD_NAME
+spec:
+  containers:
+  - name: $POD_NAME
+    image: ubuntu
+    args: ["sleep", "infinity"]
+    volumeMounts:
+    - mountPath: $PVC_PATH
+      name: $PVC_NAME
+  restartPolicy: Never
+  volumes:
+    - name: $PVC_NAME
+      persistentVolumeClaim:
+        claimName: $PVC_NAME
+EOF
+
+# create pod
+kubectl create -f $POD_FILE
+
+# wait for pod to initialize
+POD_STATUS=""
+
+while [[ $POD_STATUS != "Running" ]]; do
+	sleep 1
+	POD_STATUS="$(kubectl get pods --no-headers $POD_NAME | awk '{ print $3 }')"
+	POD_STATUS="$(echo $POD_STATUS)"
+done
+
+# copy input data to pod
+echo "copying data..."
+
+kubectl exec $POD_NAME -- bash -c "mkdir -p $PVC_PATH/$USER"
+kubectl cp "$LOCAL_PATH" "$POD_NAME:$PVC_PATH/$USER/$(basename $LOCAL_PATH)"
+
+# delete pod
+kubectl delete -f $POD_FILE
+rm -f $POD_FILE

kube/kube-save.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Save output data from a Persistent Volume on a Kubernetes cluster.
+
+# parse command-line arguments
+if [[ $# != 2 ]]; then
+	echo "usage: $0 <pvc-name> <remote-path>"
+	exit -1
+fi
+
+PVC_NAME="$1"
+PVC_PATH="/workspace"
+POD_FILE="pod.yaml"
+POD_NAME="$USER-save-$(printf %04x $RANDOM)"
+REMOTE_PATH="$2"
+
+# create pod config file
+cat > $POD_FILE <<EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  name: $POD_NAME
+spec:
+  containers:
+  - name: $POD_NAME
+    image: ubuntu
+    args: ["sleep", "infinity"]
+    volumeMounts:
+    - mountPath: $PVC_PATH
+      name: $PVC_NAME
+  restartPolicy: Never
+  volumes:
+    - name: $PVC_NAME
+      persistentVolumeClaim:
+        claimName: $PVC_NAME
+EOF
+
+# create pod
+kubectl create -f $POD_FILE
+
+# wait for pod to initialize
+POD_STATUS=""
+
+while [[ $POD_STATUS != "Running" ]]; do
+	sleep 1
+	POD_STATUS="$(kubectl get pods --no-headers $POD_NAME | awk '{ print $3 }')"
+	POD_STATUS="$(echo $POD_STATUS)"
+done
+
+# copy output data from pod
+echo "copying data..."
+
+kubectl exec $POD_NAME -- bash -c "for f in \$(find $PVC_PATH/$USER/$REMOTE_PATH -type l); do cp --remove-destination \$(readlink \$f) \$f; done"
+kubectl cp "$POD_NAME:$PVC_PATH/$USER/$REMOTE_PATH" "$(basename $REMOTE_PATH)"
+
+# delete pod
+kubectl delete -f $POD_FILE
+rm -f $POD_FILE

requirements.txt

@@ -0,0 +1,4 @@
+pandas
+scipy
+statsmodels
+sklearn