From 00148825fc35ddada2cfc59a30d904e0d8cace7c Mon Sep 17 00:00:00 2001
From: Karl Ostmo <kostmo@gmail.com>
Date: Mon, 15 Apr 2019 12:26:50 -0700
Subject: [PATCH] Run shellcheck on Jenkins scripts (#18874)

Summary:
closes #18873

Doesn't fail the build on warnings yet.
Also fix most severe shellcheck warnings
Limited to `.jenkins/pytorch/` at this time
Pull Request resolved: https://github.com/pytorch/pytorch/pull/18874

Differential Revision: D14936165

Pulled By: kostmo

fbshipit-source-id: 1ee335695e54fe6c387ef0f6606ea7011dad0fd4
---
 .jenkins/pytorch/build-asan.sh                         |  2 ++
 .jenkins/pytorch/build.sh                              |  6 ++++--
 .jenkins/pytorch/common.sh                             |  2 +-
 .jenkins/pytorch/docker-build-test.sh                  |  2 ++
 .jenkins/pytorch/macos-build.sh                        |  2 ++
 .jenkins/pytorch/macos-test.sh                         |  2 ++
 .jenkins/pytorch/multigpu-test.sh                      |  2 ++
 .jenkins/pytorch/perf_test/common.sh                   |  2 +-
 .../perf_test/test_cpu_speed_mini_sequence_labeler.sh  |  4 ++--
 .jenkins/pytorch/perf_test/test_cpu_speed_mnist.sh     |  4 ++--
 .jenkins/pytorch/perf_test/test_cpu_speed_torch.sh     |  4 +++-
 .../pytorch/perf_test/test_cpu_speed_torch_tensor.sh   |  4 +++-
 .../pytorch/perf_test/test_gpu_speed_cudnn_lstm.sh     |  4 ++--
 .jenkins/pytorch/perf_test/test_gpu_speed_lstm.sh      |  4 ++--
 .jenkins/pytorch/perf_test/test_gpu_speed_mlstm.sh     |  4 ++--
 .jenkins/pytorch/perf_test/test_gpu_speed_mnist.sh     |  4 ++--
 .../perf_test/test_gpu_speed_word_language_model.sh    |  4 ++--
 .jenkins/pytorch/short-perf-test-cpu.sh                |  7 ++++++-
 .jenkins/pytorch/short-perf-test-gpu.sh                |  2 ++
 .jenkins/pytorch/test.sh                               |  2 ++
 .jenkins/pytorch/win-build.sh                          |  1 +
 .jenkins/pytorch/win-test.sh                           |  1 +
 .jenkins/run-shellcheck.sh                             | 10 ++++++++++
 .travis.yml                                            |  4 ++++
 24 files changed, 62 insertions(+), 21 deletions(-)
 create mode 100755 .jenkins/run-shellcheck.sh

diff --git a/.jenkins/pytorch/build-asan.sh b/.jenkins/pytorch/build-asan.sh
index c9e48b1d08db23..a83d2af270705a 100755
--- a/.jenkins/pytorch/build-asan.sh
+++ b/.jenkins/pytorch/build-asan.sh
@@ -4,7 +4,9 @@
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 echo "Clang version:"
diff --git a/.jenkins/pytorch/build.sh b/.jenkins/pytorch/build.sh
index dccf98d46fcfd7..edb1272fd3e201 100755
--- a/.jenkins/pytorch/build.sh
+++ b/.jenkins/pytorch/build.sh
@@ -4,7 +4,9 @@
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 # For distributed, four environmental configs:
@@ -31,7 +33,7 @@ if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9*gcc7* ]] || [[ "$BUILD_ENVIRONMENT"
 fi
 
 if [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-xenial-py3-clang5-asan* ]]; then
-  exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" $*
+  exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" "$@"
 fi
 
 echo "Python version:"
@@ -114,7 +116,7 @@ fi
 # gcc 7 with sccache seems to have intermittent OOM issue if all cores are used
 if [ -z "$MAX_JOBS" ]; then
   if ([[ "$BUILD_ENVIRONMENT" == *cuda* ]] || [[ "$BUILD_ENVIRONMENT" == *gcc7* ]]) && which sccache > /dev/null; then
-    export MAX_JOBS=`expr $(nproc) - 1`
+    export MAX_JOBS=$(($(nproc) - 1))
   fi
 fi
 
diff --git a/.jenkins/pytorch/common.sh b/.jenkins/pytorch/common.sh
index b0ec4b521b9c33..764139c874fc6b 100644
--- a/.jenkins/pytorch/common.sh
+++ b/.jenkins/pytorch/common.sh
@@ -89,7 +89,7 @@ if which sccache > /dev/null; then
   sccache --zero-stats
   function sccache_epilogue() {
     echo '=================== sccache compilation log ==================='
-    python $(dirname "${BASH_SOURCE[0]}")/print_sccache_log.py ~/sccache_error.log
+    python "$(dirname "${BASH_SOURCE[0]}")/print_sccache_log.py" ~/sccache_error.log
     echo '=========== If your build fails, please take a look at the log above for possible reasons ==========='
     sccache --show-stats
     sccache --stop-server || true
diff --git a/.jenkins/pytorch/docker-build-test.sh b/.jenkins/pytorch/docker-build-test.sh
index 508699aa9413ad..c8fcd1a0c03f79 100755
--- a/.jenkins/pytorch/docker-build-test.sh
+++ b/.jenkins/pytorch/docker-build-test.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="docker-build-test"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 docker build -t pytorch .
diff --git a/.jenkins/pytorch/macos-build.sh b/.jenkins/pytorch/macos-build.sh
index b8ae8ea07395a5..befd0389cd387f 100755
--- a/.jenkins/pytorch/macos-build.sh
+++ b/.jenkins/pytorch/macos-build.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 export PATH="/usr/local/bin:$PATH"
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
diff --git a/.jenkins/pytorch/macos-test.sh b/.jenkins/pytorch/macos-test.sh
index d76ffafb089aee..3c6be92e7f46b9 100755
--- a/.jenkins/pytorch/macos-test.sh
+++ b/.jenkins/pytorch/macos-test.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 export PATH="/usr/local/bin:$PATH"
diff --git a/.jenkins/pytorch/multigpu-test.sh b/.jenkins/pytorch/multigpu-test.sh
index 4e2e0ec71e9c92..3d082b55e04ac1 100755
--- a/.jenkins/pytorch/multigpu-test.sh
+++ b/.jenkins/pytorch/multigpu-test.sh
@@ -4,7 +4,9 @@
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 echo "Testing pytorch (distributed only)"
diff --git a/.jenkins/pytorch/perf_test/common.sh b/.jenkins/pytorch/perf_test/common.sh
index 21ce05f722c175..a1c38e9c602413 100644
--- a/.jenkins/pytorch/perf_test/common.sh
+++ b/.jenkins/pytorch/perf_test/common.sh
@@ -10,7 +10,7 @@ get_runtime_of_command () {
   TIMEFORMAT=%R
 
   # runtime=$( { time ($@ &> /dev/null); } 2>&1 1>/dev/null)
-  runtime=$( { time $@; } 2>&1 1>/dev/null)
+  runtime=$( { time "$@"; } 2>&1 1>/dev/null)
   if [[ $runtime == *"Error"* ]]; then
     exit 1
   fi
diff --git a/.jenkins/pytorch/perf_test/test_cpu_speed_mini_sequence_labeler.sh b/.jenkins/pytorch/perf_test/test_cpu_speed_mini_sequence_labeler.sh
index e1360c76909733..795251fc862576 100644
--- a/.jenkins/pytorch/perf_test/test_cpu_speed_mini_sequence_labeler.sh
+++ b/.jenkins/pytorch/perf_test/test_cpu_speed_mini_sequence_labeler.sh
@@ -19,14 +19,14 @@ test_cpu_speed_mini_sequence_labeler () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python main.py)
     SAMPLE_ARRAY+=(${runtime})
   done
 
   cd ../../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 
diff --git a/.jenkins/pytorch/perf_test/test_cpu_speed_mnist.sh b/.jenkins/pytorch/perf_test/test_cpu_speed_mnist.sh
index 04d7e9a5491c70..3048b9f20be8e9 100644
--- a/.jenkins/pytorch/perf_test/test_cpu_speed_mnist.sh
+++ b/.jenkins/pytorch/perf_test/test_cpu_speed_mnist.sh
@@ -20,7 +20,7 @@ test_cpu_speed_mnist () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python main.py --epochs 1 --no-log)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
@@ -28,7 +28,7 @@ test_cpu_speed_mnist () {
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 
diff --git a/.jenkins/pytorch/perf_test/test_cpu_speed_torch.sh b/.jenkins/pytorch/perf_test/test_cpu_speed_torch.sh
index cd4776c9f59818..e805389aeaa87c 100644
--- a/.jenkins/pytorch/perf_test/test_cpu_speed_torch.sh
+++ b/.jenkins/pytorch/perf_test/test_cpu_speed_torch.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
 . ./common.sh
 
 test_cpu_speed_torch () {
@@ -17,7 +19,7 @@ test_cpu_speed_torch () {
   fi
 
   if ! python perf-tests/modules/test_cpu_torch.py ${ARGS}; then
-    echo "To reproduce this regression, run \`cd .jenkins/pytorch/perf_test/ && bash "${FUNCNAME[0]}".sh\` on your local machine and compare the runtime before/after your code change."
+    echo "To reproduce this regression, run \`cd .jenkins/pytorch/perf_test/ && bash ${FUNCNAME[0]}.sh\` on your local machine and compare the runtime before/after your code change."
     exit 1
   fi
 }
diff --git a/.jenkins/pytorch/perf_test/test_cpu_speed_torch_tensor.sh b/.jenkins/pytorch/perf_test/test_cpu_speed_torch_tensor.sh
index c924e2efcf9cde..6dc5a7b200b439 100644
--- a/.jenkins/pytorch/perf_test/test_cpu_speed_torch_tensor.sh
+++ b/.jenkins/pytorch/perf_test/test_cpu_speed_torch_tensor.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
 . ./common.sh
 
 test_cpu_speed_torch_tensor () {
@@ -17,7 +19,7 @@ test_cpu_speed_torch_tensor () {
   fi
 
   if ! python perf-tests/modules/test_cpu_torch_tensor.py ${ARGS}; then
-    echo "To reproduce this regression, run \`cd .jenkins/pytorch/perf_test/ && bash "${FUNCNAME[0]}".sh\` on your local machine and compare the runtime before/after your code change."
+    echo "To reproduce this regression, run \`cd .jenkins/pytorch/perf_test/ && bash ${FUNCNAME[0]}.sh\` on your local machine and compare the runtime before/after your code change."
     exit 1
   fi
 }
diff --git a/.jenkins/pytorch/perf_test/test_gpu_speed_cudnn_lstm.sh b/.jenkins/pytorch/perf_test/test_gpu_speed_cudnn_lstm.sh
index ab02eb8367fb4c..667cfba617fca0 100644
--- a/.jenkins/pytorch/perf_test/test_gpu_speed_cudnn_lstm.sh
+++ b/.jenkins/pytorch/perf_test/test_gpu_speed_cudnn_lstm.sh
@@ -19,7 +19,7 @@ test_gpu_speed_cudnn_lstm () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python cudnn_lstm.py --skip-cpu-governor-check)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
@@ -27,7 +27,7 @@ test_gpu_speed_cudnn_lstm () {
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 
diff --git a/.jenkins/pytorch/perf_test/test_gpu_speed_lstm.sh b/.jenkins/pytorch/perf_test/test_gpu_speed_lstm.sh
index ddc0d6ff741b01..ea220b33ac7cc3 100644
--- a/.jenkins/pytorch/perf_test/test_gpu_speed_lstm.sh
+++ b/.jenkins/pytorch/perf_test/test_gpu_speed_lstm.sh
@@ -19,7 +19,7 @@ test_gpu_speed_lstm () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python lstm.py --skip-cpu-governor-check)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
@@ -27,7 +27,7 @@ test_gpu_speed_lstm () {
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 
diff --git a/.jenkins/pytorch/perf_test/test_gpu_speed_mlstm.sh b/.jenkins/pytorch/perf_test/test_gpu_speed_mlstm.sh
index fd7626710d9f2d..62b94a7b21d181 100644
--- a/.jenkins/pytorch/perf_test/test_gpu_speed_mlstm.sh
+++ b/.jenkins/pytorch/perf_test/test_gpu_speed_mlstm.sh
@@ -19,7 +19,7 @@ test_gpu_speed_mlstm () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python mlstm.py --skip-cpu-governor-check)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
@@ -27,7 +27,7 @@ test_gpu_speed_mlstm () {
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 
diff --git a/.jenkins/pytorch/perf_test/test_gpu_speed_mnist.sh b/.jenkins/pytorch/perf_test/test_gpu_speed_mnist.sh
index 8827847378672b..84685419542a83 100644
--- a/.jenkins/pytorch/perf_test/test_gpu_speed_mnist.sh
+++ b/.jenkins/pytorch/perf_test/test_gpu_speed_mnist.sh
@@ -23,7 +23,7 @@ test_gpu_speed_mnist () {
   # Needs warm up to get accurate number
   python main.py --epochs 1 --no-log
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python main.py --epochs 1 --no-log)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
@@ -31,7 +31,7 @@ test_gpu_speed_mnist () {
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 
diff --git a/.jenkins/pytorch/perf_test/test_gpu_speed_word_language_model.sh b/.jenkins/pytorch/perf_test/test_gpu_speed_word_language_model.sh
index 89ed0446d95c20..b1dea09c7c3475 100644
--- a/.jenkins/pytorch/perf_test/test_gpu_speed_word_language_model.sh
+++ b/.jenkins/pytorch/perf_test/test_gpu_speed_word_language_model.sh
@@ -28,7 +28,7 @@ test_gpu_speed_word_language_model () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python main.py --cuda --epochs 1)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
@@ -36,7 +36,7 @@ test_gpu_speed_word_language_model () {
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 
diff --git a/.jenkins/pytorch/short-perf-test-cpu.sh b/.jenkins/pytorch/short-perf-test-cpu.sh
index cded7f1858b1d1..ae838276bd4d98 100755
--- a/.jenkins/pytorch/short-perf-test-cpu.sh
+++ b/.jenkins/pytorch/short-perf-test-cpu.sh
@@ -1,7 +1,12 @@
 #!/bin/bash
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="short-perf-test-cpu"
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+SCRIPT_PARENT_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+# shellcheck source=.jenkins/pytorch/common.sh
+source "$SCRIPT_PARENT_DIR/common.sh"
 
 cd .jenkins/pytorch/perf_test
 
diff --git a/.jenkins/pytorch/short-perf-test-gpu.sh b/.jenkins/pytorch/short-perf-test-gpu.sh
index df8ec018d4f520..8fd701e1972002 100755
--- a/.jenkins/pytorch/short-perf-test-gpu.sh
+++ b/.jenkins/pytorch/short-perf-test-gpu.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="short-perf-test-gpu"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 pushd .jenkins/pytorch/perf_test
diff --git a/.jenkins/pytorch/test.sh b/.jenkins/pytorch/test.sh
index 456325553d0ece..24ba4e0d28e07b 100755
--- a/.jenkins/pytorch/test.sh
+++ b/.jenkins/pytorch/test.sh
@@ -4,7 +4,9 @@
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 echo "Testing pytorch"
diff --git a/.jenkins/pytorch/win-build.sh b/.jenkins/pytorch/win-build.sh
index bf22cdfd6d0f07..3b6f330dd5b9c1 100755
--- a/.jenkins/pytorch/win-build.sh
+++ b/.jenkins/pytorch/win-build.sh
@@ -9,6 +9,7 @@ if [ ! -f setup.py ]; then
   exit 1
 fi
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME=pytorch-win-ws2016-cuda9-cudnn7-py3-build
 
 SCRIPT_PARENT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
diff --git a/.jenkins/pytorch/win-test.sh b/.jenkins/pytorch/win-test.sh
index 6af24ff9d81d08..c51066622684a4 100755
--- a/.jenkins/pytorch/win-test.sh
+++ b/.jenkins/pytorch/win-test.sh
@@ -1,5 +1,6 @@
 #!/bin/bash -ex
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME=pytorch-win-ws2016-cuda9-cudnn7-py3-test
 
 SCRIPT_PARENT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
diff --git a/.jenkins/run-shellcheck.sh b/.jenkins/run-shellcheck.sh
new file mode 100755
index 00000000000000..1333e9ab6f49dd
--- /dev/null
+++ b/.jenkins/run-shellcheck.sh
@@ -0,0 +1,10 @@
+#!/bin/bash -xe
+
+# One may want to invoke this script locally as follows:
+#
+#   .jenkins/run-shellcheck.sh --color=always | less -R
+
+
+EXCLUSIONS=SC2086,SC1091,SC2155,SC1090,SC2164,SC1003
+
+find .jenkins/pytorch -name *.sh | xargs shellcheck --exclude=$EXCLUSIONS --external-sources "$@" || true
diff --git a/.travis.yml b/.travis.yml
index beb0d69545ea0c..75c2a286925d6c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,6 +16,10 @@ matrix:
         python: "3.6"
         dist: xenial
         script: cd .circleci && ./ensure-consistency.py
+      - name: "Shellcheck Jenkins scripts"
+        dist: xenial
+        install: sudo apt-get install -y shellcheck
+        script: .jenkins/run-shellcheck.sh
       - name: "Ensure no tabs"
         python: "2.7"
         script: