diff --git a/CHANGES.txt b/CHANGES.txt index 1c0caf2..5897cd3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,7 @@ -1.1.0 - 2015-xx-xx +1.1.1 - 2015-xx-xx + + +1.1.0 - 2015-01-22 1. Added plugin for deploying Ambari/HDP with: ./bdutil -e platforms/hdp/ambari_env.sh deploy @@ -16,6 +19,7 @@ with compiler/tools instead of just the minimal JRE; set to 'true' in single_node_env.sh and ambari_env.sh. 8. Added python script to allocate memory more intelligently in Hadoop 2. + 9. Upgraded default Hadoop 2 version to 2.5.2. 1.0.1 - 2014-12-16 diff --git a/bdutil b/bdutil index f676589..ef94c75 100755 --- a/bdutil +++ b/bdutil @@ -20,7 +20,7 @@ # # Usage: ./bdutil [deploy|delete] [optional: ]... -BDUTIL_VERSION='{{ version_string }}' +BDUTIL_VERSION='1.1.1-SNAPSHOT' # Prints the usage for this script and exits. function print_usage() { diff --git a/bdutil_env.sh b/bdutil_env.sh index bbd17e2..41aea7a 100644 --- a/bdutil_env.sh +++ b/bdutil_env.sh @@ -167,13 +167,13 @@ CORES_PER_REDUCE_TASK=1.0 JAVAOPTS='-Xms1024m -Xmx2048m' # Complete URL for downloading the GCS Connector JAR file. -GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/gcs/gcs-connector-1.3.2-SNAPSHOT-hadoop1-20141222-135330.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1450821786&Signature=Y8CR1WVu2lOMdS9mgXXhIAELvhrGvmO%2BBrooLjWWRJWJCPBQ3MT%2FdVVK%2BnSqdTeMdK8jycRV6KuTPMs%2Bmr6yxHvhucKEc14mZcIdVyEo9FiS46QQeD3cn1GkKrFt90rfQ%2FJC7KaJQcqyYlZLN0lb40n8dSPPL994xbx0Lqf7lF0=' +GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-1.3.2-hadoop1.jar' # Complete URL for downloading the BigQuery Connector JAR file. -BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/bigquery/bigquery-connector-0.5.1-SNAPSHOT-20141222-112706-hadoop1.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1450812947&Signature=O514AO229YtAaQIwLgmyV5ILuebndeT4g%2FJw2TvPWrMO7vK42XscpIllMi%2B128E5Q50jzGHlUxZaRNc37yRXLylekSo88WOTg9YUxN%2FjLzYmIkOlFPUfWLogjUVvlh4zvGYqaZ%2FpbNW2DWtQM7Yhy1khFgD0%2B51ruCgjfMHzVz0=' +BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/bigquery/bigquery-connector-0.5.1-hadoop1.jar' # Complete URL for downloading the Cloud Datastore Connector JAR file. -DATASTORE_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/datastore/datastore-connector-0.14.10-SNAPSHOT-20141222-133725-hadoop1.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1450820611&Signature=mPGE%2Fv%2FMfV5c1AY%2BjDiJ9troPcZMUzSaw82RdDXRj8Tzxh5JuGwDixGziaAXM0aW9pRb%2B8X97hAICM2vo%2B4lVHg6JJtyXhhGYJfn8o9KFp8x%2FoMp7AiGWlqIFvM27M%2FM4TZTpu%2BzWDtm9WTXWi4SEpOuCbZoqTu2fRjFuNTOpfk=' +DATASTORE_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/datastore/datastore-connector-0.14.9-hadoop1.jar' # Complete URL for downloading the configuration script. BDCONFIG='https://storage.googleapis.com/hadoop-tools/bdconfig/bdconfig-0.28.1.tar.gz' diff --git a/hadoop2_env.sh b/hadoop2_env.sh index 346d0ca..13c003c 100644 --- a/hadoop2_env.sh +++ b/hadoop2_env.sh @@ -44,11 +44,11 @@ CORES_PER_REDUCE_TASK=2.0 CORES_PER_APP_MASTER=2.0 # Connector with Hadoop AbstractFileSystem implemenation for YARN -GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/gcs/gcs-connector-1.3.2-SNAPSHOT-hadoop2-20141222-135330.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1450821787&Signature=cQDn6fvv2Hyu58z9uzLQEIu42WyeSOY97lO8pXgZNq%2BEX5YGW8VjBWP1YzBX1Slkufc6USrb4009TZhSu%2BQATsZwYa7jOGh0sLSsHEmCPvUI3HrENag8rvvhefLnxGx6vrFcA2fOskivUOp5ZTyh1K0sCDBqjoSLTbGIFXeuNr0=' +GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-1.3.2-hadoop2.jar' -DATASTORE_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/datastore/datastore-connector-0.14.10-SNAPSHOT-20141222-133725-hadoop2.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1450820611&Signature=iZWtERtR1vRszg8D734aO%2BadLcOpSjgOF5%2BxsNYLN%2B26oVO8KnEAF%2FRFVEsVQMXxwMSY8olp7r7zegBhOsuAflm%2F7d8rSpXOqWuoUI7VfGaadC%2FNfkQTux%2Fl0rkvZj1bk85fsQO115p1voOyIH%2FweZ8U05cx6hIkvdG3s%2FPnVts=' +DATASTORE_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/datastore/datastore-connector-0.14.9-hadoop2.jar' -BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/bigquery/bigquery-connector-0.5.1-SNAPSHOT-20141222-112706-hadoop2.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1450812947&Signature=M14Jcxt5HjiCGhDvOQGJdr9UcwrFp4ELJ617AW9O2cpZ453%2F5irwsnHPgGjeczpoD4sCxidheWTCEcSOa8y0E163zBNKXda1Nc0gHyMVQMQOPWzfQ1QgQCTuT%2BD39FjNyvOazlXJymzuRJruLRK2CY2HbbN%2FSpacVUCrckn6ABU=' +BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/bigquery/bigquery-connector-0.5.1-hadoop2.jar' HDFS_DATA_DIRS_PERM='700' diff --git a/libexec/configure_hadoop.sh b/libexec/configure_hadoop.sh index fd7ff46..f4ae381 100644 --- a/libexec/configure_hadoop.sh +++ b/libexec/configure_hadoop.sh @@ -53,6 +53,12 @@ TOTAL_MEM=$(free -m | awk '/^Mem:/{print $2}') HADOOP_MR_MASTER_MEM_MB=$(python -c "print int(${TOTAL_MEM} * \ ${HADOOP_MASTER_MAPREDUCE_MEMORY_FRACTION})") +# Fix Python 2.6 on CentOS +# TODO(user): Extract this into a helper. +if ! python -c 'import argparse' && [[ -x $(which yum) ]]; then + yum install -y python-argparse +fi + # MapReduce v2 (and YARN) Configuration if [[ -x configure_mrv2_mem.py ]]; then TEMP_ENV_FILE=$(mktemp /tmp/mrv2_XXX_tmp_env.sh) diff --git a/samples/bigquery_wordcount.jar b/samples/bigquery_wordcount.jar index 46df9f7..c1f4271 100644 Binary files a/samples/bigquery_wordcount.jar and b/samples/bigquery_wordcount.jar differ diff --git a/samples/datastore_wordcount.jar b/samples/datastore_wordcount.jar index 8100a84..bab8707 100644 Binary files a/samples/datastore_wordcount.jar and b/samples/datastore_wordcount.jar differ diff --git a/samples/datastore_wordcountsetup.jar b/samples/datastore_wordcountsetup.jar index 4eb9c1f..393f93c 100644 Binary files a/samples/datastore_wordcountsetup.jar and b/samples/datastore_wordcountsetup.jar differ diff --git a/samples/datastoretobigquery_wordcount.jar b/samples/datastoretobigquery_wordcount.jar index 923f7d9..bfd790c 100644 Binary files a/samples/datastoretobigquery_wordcount.jar and b/samples/datastoretobigquery_wordcount.jar differ