Merge pull request #16 from BU-Spark/running-ml-on-cluster-jepstein6.…

…11.24 Jobfile for running ML on cluster
BU-Spark · Jun 18, 2024 · 5c91dd7 · 5c91dd7
2 parents 5adfcb6 + cee83fd
commit 5c91dd7
Show file tree

Hide file tree

Showing 2 changed files with 56 additions and 18 deletions.
diff --git a/data cleaning.ipynb b/data cleaning.ipynb
@@ -12,22 +12,22 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Requirement already satisfied: geopandas in /opt/miniconda3/lib/python3.12/site-packages (0.14.4)\n",
-      "Requirement already satisfied: shapely in /opt/miniconda3/lib/python3.12/site-packages (2.0.4)\n",
-      "Requirement already satisfied: fiona>=1.8.21 in /opt/miniconda3/lib/python3.12/site-packages (from geopandas) (1.9.6)\n",
-      "Requirement already satisfied: numpy>=1.22 in /opt/miniconda3/lib/python3.12/site-packages (from geopandas) (1.26.4)\n",
-      "Requirement already satisfied: packaging in /opt/miniconda3/lib/python3.12/site-packages (from geopandas) (23.2)\n",
-      "Requirement already satisfied: pandas>=1.4.0 in /opt/miniconda3/lib/python3.12/site-packages (from geopandas) (2.2.1)\n",
-      "Requirement already satisfied: pyproj>=3.3.0 in /opt/miniconda3/lib/python3.12/site-packages (from geopandas) (3.6.1)\n",
-      "Requirement already satisfied: attrs>=19.2.0 in /opt/miniconda3/lib/python3.12/site-packages (from fiona>=1.8.21->geopandas) (23.2.0)\n",
-      "Requirement already satisfied: certifi in /opt/miniconda3/lib/python3.12/site-packages (from fiona>=1.8.21->geopandas) (2024.6.2)\n",
-      "Requirement already satisfied: click~=8.0 in /opt/miniconda3/lib/python3.12/site-packages (from fiona>=1.8.21->geopandas) (8.1.7)\n",
-      "Requirement already satisfied: click-plugins>=1.0 in /opt/miniconda3/lib/python3.12/site-packages (from fiona>=1.8.21->geopandas) (1.1.1)\n",
-      "Requirement already satisfied: cligj>=0.5 in /opt/miniconda3/lib/python3.12/site-packages (from fiona>=1.8.21->geopandas) (0.7.2)\n",
-      "Requirement already satisfied: six in /opt/miniconda3/lib/python3.12/site-packages (from fiona>=1.8.21->geopandas) (1.16.0)\n",
-      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/miniconda3/lib/python3.12/site-packages (from pandas>=1.4.0->geopandas) (2.9.0.post0)\n",
-      "Requirement already satisfied: pytz>=2020.1 in /opt/miniconda3/lib/python3.12/site-packages (from pandas>=1.4.0->geopandas) (2024.1)\n",
-      "Requirement already satisfied: tzdata>=2022.7 in /opt/miniconda3/lib/python3.12/site-packages (from pandas>=1.4.0->geopandas) (2023.3)\n"
+      "Requirement already satisfied: geopandas in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (0.14.4)\n",
+      "Requirement already satisfied: shapely in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (2.0.4)\n",
+      "Requirement already satisfied: fiona>=1.8.21 in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from geopandas) (1.9.6)\n",
+      "Requirement already satisfied: numpy>=1.22 in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from geopandas) (1.26.4)\n",
+      "Requirement already satisfied: packaging in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from geopandas) (23.2)\n",
+      "Requirement already satisfied: pandas>=1.4.0 in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from geopandas) (2.1.1)\n",
+      "Requirement already satisfied: pyproj>=3.3.0 in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from geopandas) (3.6.1)\n",
+      "Requirement already satisfied: attrs>=19.2.0 in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from fiona>=1.8.21->geopandas) (23.2.0)\n",
+      "Requirement already satisfied: certifi in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from fiona>=1.8.21->geopandas) (2024.6.2)\n",
+      "Requirement already satisfied: click~=8.0 in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from fiona>=1.8.21->geopandas) (8.1.7)\n",
+      "Requirement already satisfied: click-plugins>=1.0 in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from fiona>=1.8.21->geopandas) (1.1.1)\n",
+      "Requirement already satisfied: cligj>=0.5 in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from fiona>=1.8.21->geopandas) (0.7.2)\n",
+      "Requirement already satisfied: six in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from fiona>=1.8.21->geopandas) (1.16.0)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2.9.0)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2024.1)\n",
+      "Requirement already satisfied: tzdata>=2022.1 in /opt/anaconda3/envs/cs589/lib/python3.10/site-packages (from pandas>=1.4.0->geopandas) (2024.1)\n"
      ]
     }
    ],
@@ -749,7 +749,9 @@
    "cell_type": "code",
    "execution_count": 11,
    "id": "b40d5a8a-9611-4889-b092-0954d7280ef4",
-   "metadata": {},
+   "metadata": {
+    "scrolled": true
+   },
    "outputs": [
     {
      "name": "stdout",
@@ -2274,7 +2276,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.2"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

diff --git a/jobfiles/isolation_trees_job.qsub b/jobfiles/isolation_trees_job.qsub
@@ -0,0 +1,36 @@
+#!/bin/bash -l
+
+# Submit job with the command: qsub job.qsub
+
+# Set the runtime limit (default 12 hours):
+#$ -l h_rt=12:00:00
+
+# Send an email when the job finishes or if it is aborted (by default no email is sent).
+#$ -m ea
+
+# Give the job a name (default: script name)
+#$ -N one_gpu
+
+# Set SCC project
+#$ -P sparkpit
+
+# Combine output and error files into a single file
+#$ -j y
+
+# Request 8 Cores
+#$ -pe omp 8
+
+# Keep track of information related to the current job
+echo "=========================================================="
+echo "Start date : $(date)"
+echo "Job name : $JOB_NAME"
+echo "Job ID : $JOB_ID  $SGE_TASK_ID"
+echo "=========================================================="
+
+# convert a notebook to a python script and run
+module load miniconda/23.11.0 && cd /rprojectnb/sparkpit/pit-seasonwatch && conda activate seasonwatch-ml && jupyter nbconvert --to script NOTEBOOK_NAME.ipynb && python NOTEBOOK_NAME.py
+
+# or...
+
+# run a python file
+module load miniconda/23.11.0 && cd /rprojectnb/sparkpit/pit-seasonwatch && conda activate seasonwatch-ml && python SCRIPTNAME.py