Release of version 1.1.0

GoogleCloudDataproc · Jan 26, 2015 · 29da977 · 29da977
1 parent daf2b46
commit 29da977
Show file tree

Hide file tree

Showing 9 changed files with 18 additions and 8 deletions.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -1,4 +1,7 @@
-1.1.0 - 2015-xx-xx
+1.1.1 - 2015-xx-xx
+
+
+1.1.0 - 2015-01-22
 
   1. Added plugin for deploying Ambari/HDP with:
          ./bdutil -e platforms/hdp/ambari_env.sh deploy
@@ -16,6 +19,7 @@
      with compiler/tools instead of just the minimal JRE; set to 'true' in
      single_node_env.sh and ambari_env.sh.
   8. Added python script to allocate memory more intelligently in Hadoop 2.
+  9. Upgraded default Hadoop 2 version to 2.5.2.
 
 
 1.0.1 - 2014-12-16

diff --git a/bdutil b/bdutil
@@ -20,7 +20,7 @@
 #
 # Usage: ./bdutil [deploy|delete] [optional: <paths to env files>]...
 
-BDUTIL_VERSION='{{ version_string }}'
+BDUTIL_VERSION='1.1.1-SNAPSHOT'
 
 # Prints the usage for this script and exits.
 function print_usage() {

diff --git a/bdutil_env.sh b/bdutil_env.sh
@@ -167,13 +167,13 @@ CORES_PER_REDUCE_TASK=1.0
 JAVAOPTS='-Xms1024m -Xmx2048m'
 
 # Complete URL for downloading the GCS Connector JAR file.
-GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/gcs/gcs-connector-1.3.2-SNAPSHOT-hadoop1-20141222-135330.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1450821786&Signature=Y8CR1WVu2lOMdS9mgXXhIAELvhrGvmO%2BBrooLjWWRJWJCPBQ3MT%2FdVVK%2BnSqdTeMdK8jycRV6KuTPMs%2Bmr6yxHvhucKEc14mZcIdVyEo9FiS46QQeD3cn1GkKrFt90rfQ%2FJC7KaJQcqyYlZLN0lb40n8dSPPL994xbx0Lqf7lF0='
+GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-1.3.2-hadoop1.jar'
 
 # Complete URL for downloading the BigQuery Connector JAR file.
-BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/bigquery/bigquery-connector-0.5.1-SNAPSHOT-20141222-112706-hadoop1.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1450812947&Signature=O514AO229YtAaQIwLgmyV5ILuebndeT4g%2FJw2TvPWrMO7vK42XscpIllMi%2B128E5Q50jzGHlUxZaRNc37yRXLylekSo88WOTg9YUxN%2FjLzYmIkOlFPUfWLogjUVvlh4zvGYqaZ%2FpbNW2DWtQM7Yhy1khFgD0%2B51ruCgjfMHzVz0='
+BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/bigquery/bigquery-connector-0.5.1-hadoop1.jar'
 
 # Complete URL for downloading the Cloud Datastore Connector JAR file.
-DATASTORE_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/datastore/datastore-connector-0.14.10-SNAPSHOT-20141222-133725-hadoop1.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1450820611&Signature=mPGE%2Fv%2FMfV5c1AY%2BjDiJ9troPcZMUzSaw82RdDXRj8Tzxh5JuGwDixGziaAXM0aW9pRb%2B8X97hAICM2vo%2B4lVHg6JJtyXhhGYJfn8o9KFp8x%2FoMp7AiGWlqIFvM27M%2FM4TZTpu%2BzWDtm9WTXWi4SEpOuCbZoqTu2fRjFuNTOpfk='
+DATASTORE_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/datastore/datastore-connector-0.14.9-hadoop1.jar'
 
 # Complete URL for downloading the configuration script.
 BDCONFIG='https://storage.googleapis.com/hadoop-tools/bdconfig/bdconfig-0.28.1.tar.gz'

diff --git a/hadoop2_env.sh b/hadoop2_env.sh
@@ -44,11 +44,11 @@ CORES_PER_REDUCE_TASK=2.0
 CORES_PER_APP_MASTER=2.0
 
 # Connector with Hadoop AbstractFileSystem implemenation for YARN
-GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/gcs/gcs-connector-1.3.2-SNAPSHOT-hadoop2-20141222-135330.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1450821787&Signature=cQDn6fvv2Hyu58z9uzLQEIu42WyeSOY97lO8pXgZNq%2BEX5YGW8VjBWP1YzBX1Slkufc6USrb4009TZhSu%2BQATsZwYa7jOGh0sLSsHEmCPvUI3HrENag8rvvhefLnxGx6vrFcA2fOskivUOp5ZTyh1K0sCDBqjoSLTbGIFXeuNr0='
+GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-1.3.2-hadoop2.jar'
 
-DATASTORE_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/datastore/datastore-connector-0.14.10-SNAPSHOT-20141222-133725-hadoop2.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1450820611&Signature=iZWtERtR1vRszg8D734aO%2BadLcOpSjgOF5%2BxsNYLN%2B26oVO8KnEAF%2FRFVEsVQMXxwMSY8olp7r7zegBhOsuAflm%2F7d8rSpXOqWuoUI7VfGaadC%2FNfkQTux%2Fl0rkvZj1bk85fsQO115p1voOyIH%2FweZ8U05cx6hIkvdG3s%2FPnVts='
+DATASTORE_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/datastore/datastore-connector-0.14.9-hadoop2.jar'
 
-BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/bigquery/bigquery-connector-0.5.1-SNAPSHOT-20141222-112706-hadoop2.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1450812947&Signature=M14Jcxt5HjiCGhDvOQGJdr9UcwrFp4ELJ617AW9O2cpZ453%2F5irwsnHPgGjeczpoD4sCxidheWTCEcSOa8y0E163zBNKXda1Nc0gHyMVQMQOPWzfQ1QgQCTuT%2BD39FjNyvOazlXJymzuRJruLRK2CY2HbbN%2FSpacVUCrckn6ABU='
+BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/bigquery/bigquery-connector-0.5.1-hadoop2.jar'
 
 
 HDFS_DATA_DIRS_PERM='700'

diff --git a/libexec/configure_hadoop.sh b/libexec/configure_hadoop.sh
@@ -53,6 +53,12 @@ TOTAL_MEM=$(free -m | awk '/^Mem:/{print $2}')
 HADOOP_MR_MASTER_MEM_MB=$(python -c "print int(${TOTAL_MEM} * \
     ${HADOOP_MASTER_MAPREDUCE_MEMORY_FRACTION})")
 
+# Fix Python 2.6 on CentOS
+# TODO(user): Extract this into a helper.
+if ! python -c 'import argparse' && [[ -x $(which yum) ]]; then
+  yum install -y python-argparse
+fi
+
 # MapReduce v2 (and YARN) Configuration
 if [[ -x configure_mrv2_mem.py ]]; then
   TEMP_ENV_FILE=$(mktemp /tmp/mrv2_XXX_tmp_env.sh)

diff --git a/samples/bigquery_wordcount.jar b/samples/bigquery_wordcount.jar
diff --git a/samples/datastore_wordcount.jar b/samples/datastore_wordcount.jar
diff --git a/samples/datastore_wordcountsetup.jar b/samples/datastore_wordcountsetup.jar
diff --git a/samples/datastoretobigquery_wordcount.jar b/samples/datastoretobigquery_wordcount.jar