Run shellcheck on Jenkins scripts (pytorch#18874)

Summary: closes pytorch#18873 Doesn't fail the build on warnings yet. Also fix most severe shellcheck warnings Limited to `.jenkins/pytorch/` at this time Pull Request resolved: pytorch#18874 Differential Revision: D14936165 Pulled By: kostmo fbshipit-source-id: 1ee335695e54fe6c387ef0f6606ea7011dad0fd4
gchanan · Apr 15, 2019 · 0014882 · 0014882
1 parent a0263ec
commit 0014882
Show file tree

Hide file tree

Showing 24 changed files with 62 additions and 21 deletions.
diff --git a/.jenkins/pytorch/build-asan.sh b/.jenkins/pytorch/build-asan.sh
@@ -4,7 +4,9 @@
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 echo "Clang version:"

diff --git a/.jenkins/pytorch/build.sh b/.jenkins/pytorch/build.sh
@@ -4,7 +4,9 @@
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 # For distributed, four environmental configs:
@@ -31,7 +33,7 @@ if [[ "$BUILD_ENVIRONMENT" == *-xenial-cuda9*gcc7* ]] || [[ "$BUILD_ENVIRONMENT"
 fi
 
 if [[ "$BUILD_ENVIRONMENT" == *pytorch-linux-xenial-py3-clang5-asan* ]]; then
-  exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" $*
+  exec "$(dirname "${BASH_SOURCE[0]}")/build-asan.sh" "$@"
 fi
 
 echo "Python version:"
@@ -114,7 +116,7 @@ fi
 # gcc 7 with sccache seems to have intermittent OOM issue if all cores are used
 if [ -z "$MAX_JOBS" ]; then
   if ([[ "$BUILD_ENVIRONMENT" == *cuda* ]] || [[ "$BUILD_ENVIRONMENT" == *gcc7* ]]) && which sccache > /dev/null; then
-    export MAX_JOBS=`expr $(nproc) - 1`
+    export MAX_JOBS=$(($(nproc) - 1))
   fi
 fi
 

diff --git a/.jenkins/pytorch/common.sh b/.jenkins/pytorch/common.sh
@@ -89,7 +89,7 @@ if which sccache > /dev/null; then
   sccache --zero-stats
   function sccache_epilogue() {
     echo '=================== sccache compilation log ==================='
-    python $(dirname "${BASH_SOURCE[0]}")/print_sccache_log.py ~/sccache_error.log
+    python "$(dirname "${BASH_SOURCE[0]}")/print_sccache_log.py" ~/sccache_error.log
     echo '=========== If your build fails, please take a look at the log above for possible reasons ==========='
     sccache --show-stats
     sccache --stop-server || true

diff --git a/.jenkins/pytorch/docker-build-test.sh b/.jenkins/pytorch/docker-build-test.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="docker-build-test"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 docker build -t pytorch .
diff --git a/.jenkins/pytorch/macos-build.sh b/.jenkins/pytorch/macos-build.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 export PATH="/usr/local/bin:$PATH"
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 

diff --git a/.jenkins/pytorch/macos-test.sh b/.jenkins/pytorch/macos-test.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 export PATH="/usr/local/bin:$PATH"

diff --git a/.jenkins/pytorch/multigpu-test.sh b/.jenkins/pytorch/multigpu-test.sh
@@ -4,7 +4,9 @@
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 echo "Testing pytorch (distributed only)"

diff --git a/.jenkins/pytorch/perf_test/common.sh b/.jenkins/pytorch/perf_test/common.sh
@@ -10,7 +10,7 @@ get_runtime_of_command () {
   TIMEFORMAT=%R
 
   # runtime=$( { time ($@ &> /dev/null); } 2>&1 1>/dev/null)
-  runtime=$( { time $@; } 2>&1 1>/dev/null)
+  runtime=$( { time "$@"; } 2>&1 1>/dev/null)
   if [[ $runtime == *"Error"* ]]; then
     exit 1
   fi

diff --git a/.jenkins/pytorch/perf_test/test_cpu_speed_mini_sequence_labeler.sh b/.jenkins/pytorch/perf_test/test_cpu_speed_mini_sequence_labeler.sh
@@ -19,14 +19,14 @@ test_cpu_speed_mini_sequence_labeler () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python main.py)
     SAMPLE_ARRAY+=(${runtime})
   done
 
   cd ../../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

diff --git a/.jenkins/pytorch/perf_test/test_cpu_speed_mnist.sh b/.jenkins/pytorch/perf_test/test_cpu_speed_mnist.sh
@@ -20,15 +20,15 @@ test_cpu_speed_mnist () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python main.py --epochs 1 --no-log)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
   done
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

diff --git a/.jenkins/pytorch/perf_test/test_cpu_speed_torch.sh b/.jenkins/pytorch/perf_test/test_cpu_speed_torch.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
 . ./common.sh
 
 test_cpu_speed_torch () {
@@ -17,7 +19,7 @@ test_cpu_speed_torch () {
   fi
 
   if ! python perf-tests/modules/test_cpu_torch.py ${ARGS}; then
-    echo "To reproduce this regression, run \`cd .jenkins/pytorch/perf_test/ && bash "${FUNCNAME[0]}".sh\` on your local machine and compare the runtime before/after your code change."
+    echo "To reproduce this regression, run \`cd .jenkins/pytorch/perf_test/ && bash ${FUNCNAME[0]}.sh\` on your local machine and compare the runtime before/after your code change."
     exit 1
   fi
 }

diff --git a/.jenkins/pytorch/perf_test/test_cpu_speed_torch_tensor.sh b/.jenkins/pytorch/perf_test/test_cpu_speed_torch_tensor.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
 . ./common.sh
 
 test_cpu_speed_torch_tensor () {
@@ -17,7 +19,7 @@ test_cpu_speed_torch_tensor () {
   fi
 
   if ! python perf-tests/modules/test_cpu_torch_tensor.py ${ARGS}; then
-    echo "To reproduce this regression, run \`cd .jenkins/pytorch/perf_test/ && bash "${FUNCNAME[0]}".sh\` on your local machine and compare the runtime before/after your code change."
+    echo "To reproduce this regression, run \`cd .jenkins/pytorch/perf_test/ && bash ${FUNCNAME[0]}.sh\` on your local machine and compare the runtime before/after your code change."
     exit 1
   fi
 }

diff --git a/.jenkins/pytorch/perf_test/test_gpu_speed_cudnn_lstm.sh b/.jenkins/pytorch/perf_test/test_gpu_speed_cudnn_lstm.sh
@@ -19,15 +19,15 @@ test_gpu_speed_cudnn_lstm () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python cudnn_lstm.py --skip-cpu-governor-check)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
   done
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

diff --git a/.jenkins/pytorch/perf_test/test_gpu_speed_lstm.sh b/.jenkins/pytorch/perf_test/test_gpu_speed_lstm.sh
@@ -19,15 +19,15 @@ test_gpu_speed_lstm () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python lstm.py --skip-cpu-governor-check)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
   done
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

diff --git a/.jenkins/pytorch/perf_test/test_gpu_speed_mlstm.sh b/.jenkins/pytorch/perf_test/test_gpu_speed_mlstm.sh
@@ -19,15 +19,15 @@ test_gpu_speed_mlstm () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python mlstm.py --skip-cpu-governor-check)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
   done
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

diff --git a/.jenkins/pytorch/perf_test/test_gpu_speed_mnist.sh b/.jenkins/pytorch/perf_test/test_gpu_speed_mnist.sh
@@ -23,15 +23,15 @@ test_gpu_speed_mnist () {
   # Needs warm up to get accurate number
   python main.py --epochs 1 --no-log
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python main.py --epochs 1 --no-log)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
   done
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

diff --git a/.jenkins/pytorch/perf_test/test_gpu_speed_word_language_model.sh b/.jenkins/pytorch/perf_test/test_gpu_speed_word_language_model.sh
@@ -28,15 +28,15 @@ test_gpu_speed_word_language_model () {
   SAMPLE_ARRAY=()
   NUM_RUNS=$1
 
-  for (( i=1; i<=$NUM_RUNS; i++ )) do
+  for (( i=1; i<=NUM_RUNS; i++ )) do
     runtime=$(get_runtime_of_command python main.py --cuda --epochs 1)
     echo $runtime
     SAMPLE_ARRAY+=(${runtime})
   done
 
   cd ../..
 
-  stats=$(python ../get_stats.py ${SAMPLE_ARRAY[@]})
+  stats=$(python ../get_stats.py "${SAMPLE_ARRAY[@]}")
   echo "Runtime stats in seconds:"
   echo $stats
 

diff --git a/.jenkins/pytorch/short-perf-test-cpu.sh b/.jenkins/pytorch/short-perf-test-cpu.sh
@@ -1,7 +1,12 @@
 #!/bin/bash
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="short-perf-test-cpu"
-source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
+
+SCRIPT_PARENT_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+# shellcheck source=.jenkins/pytorch/common.sh
+source "$SCRIPT_PARENT_DIR/common.sh"
 
 cd .jenkins/pytorch/perf_test
 

diff --git a/.jenkins/pytorch/short-perf-test-gpu.sh b/.jenkins/pytorch/short-perf-test-gpu.sh
@@ -1,6 +1,8 @@
 #!/bin/bash
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="short-perf-test-gpu"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 pushd .jenkins/pytorch/perf_test

diff --git a/.jenkins/pytorch/test.sh b/.jenkins/pytorch/test.sh
@@ -4,7 +4,9 @@
 # (This is set by default in the Docker images we build, so you don't
 # need to set it yourself.
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME="${BUILD_ENVIRONMENT}"
+
 source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
 
 echo "Testing pytorch"

diff --git a/.jenkins/pytorch/win-build.sh b/.jenkins/pytorch/win-build.sh
@@ -9,6 +9,7 @@ if [ ! -f setup.py ]; then
   exit 1
 fi
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME=pytorch-win-ws2016-cuda9-cudnn7-py3-build
 
 SCRIPT_PARENT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )

diff --git a/.jenkins/pytorch/win-test.sh b/.jenkins/pytorch/win-test.sh
@@ -1,5 +1,6 @@
 #!/bin/bash -ex
 
+# shellcheck disable=SC2034
 COMPACT_JOB_NAME=pytorch-win-ws2016-cuda9-cudnn7-py3-test
 
 SCRIPT_PARENT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )

diff --git a/.jenkins/run-shellcheck.sh b/.jenkins/run-shellcheck.sh
@@ -0,0 +1,10 @@
+#!/bin/bash -xe
+
+# One may want to invoke this script locally as follows:
+#
+#   .jenkins/run-shellcheck.sh --color=always | less -R
+
+
+EXCLUSIONS=SC2086,SC1091,SC2155,SC1090,SC2164,SC1003
+
+find .jenkins/pytorch -name *.sh | xargs shellcheck --exclude=$EXCLUSIONS --external-sources "$@" || true
diff --git a/.travis.yml b/.travis.yml
@@ -16,6 +16,10 @@ matrix:
         python: "3.6"
         dist: xenial
         script: cd .circleci && ./ensure-consistency.py
+      - name: "Shellcheck Jenkins scripts"
+        dist: xenial
+        install: sudo apt-get install -y shellcheck
+        script: .jenkins/run-shellcheck.sh
       - name: "Ensure no tabs"
         python: "2.7"
         script:
-Original file line number
+Diff line change
@@ Expand Up / @@ -9,6 +9,7 @@ if [ ! -f setup.py ]; then @@
       exit 1
     fi
+    # shellcheck disable=SC2034
     COMPACT_JOB_NAME=pytorch-win-ws2016-cuda9-cudnn7-py3-build
     SCRIPT_PARENT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
@@ Expand Down @@