New version numbers for 2015-05-27 release, updated CHANGES.txt notes.

------------- Created by MOE: http://code.google.com/p/moe-java MOE_MIGRATED_REVID=94521332
GoogleCloudDataproc · May 27, 2015 · ac8d0b0 · ac8d0b0
1 parent 11cf8be
commit ac8d0b0
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 8 deletions.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -1,3 +1,19 @@
+1.3.0 - 2015-05-27
+
+  1. Upgraded default Hadoop 2 version to 2.6.0.
+  2. Added support for making a portion of worker VMs run as "preemptible vms"
+     by setting --preemptible or PREEMPTIBLE_FRACTION to a value between
+     0.0 and 1.0 to specify the fraction of workers to run as preemptible.
+  3. Added support for deploying onto ubuntu-12-04 or ubuntu-14-04 images.
+  4. Added support for specifying boot disk sizes via --master_boot_disk_size_gb
+     and --worker_boot_disk_size_gb or MASTER_BOOT_DISK_SIZE_GB and
+     WORKER_BOOT_DISK_SIZE_GB; uses default derived from base image if unset.
+  5. Upgraded default Spark version to 1.3.1.
+  6. Removed datastore-connector installation options and samples; the connector
+     has been deprecated since February 17th, 2015. For alternatives see:
+     https://groups.google.com/forum/#!topic/gcp-hadoop-announce/D3_OZuqn4_o
+
+
 1.2.1 - 2015-05-05
 
   1. Install Java JDK with Spark; this allows spark-sql to correctly run

diff --git a/bdutil b/bdutil
@@ -20,7 +20,7 @@
 #
 # Usage: ./bdutil [deploy|delete] [optional: <paths to env files>]...
 
-BDUTIL_VERSION='1.2.1'
+BDUTIL_VERSION='1.3.0'
 
 # Prints the usage for this script and exits.
 function print_usage() {

diff --git a/bdutil_env.sh b/bdutil_env.sh
@@ -192,10 +192,10 @@ CORES_PER_REDUCE_TASK=1.0
 JAVAOPTS='-Xms1024m -Xmx2048m'
 
 # Complete URL for downloading the GCS Connector JAR file.
-GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/gcs/gcs-connector-1.3.4-SNAPSHOT-hadoop1-20150521-171734.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1463791001&Signature=f0kMCb4p737zcgr6BdhybsCSArMQ9EMc2cKhT%2FrSfCg50VbUQflEmBIchnobFnwks1BmS0SFeu%2Flj3PjTCe34aHN0zjAaWkxGJA4cMp267IBYy6sx3Qj6yE8CYVYV2D7DTRP8qTksnrkyy%2Fc9qpCwWg%2FARJ7EKGMczloRgTk8xo='
+GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-1.4.0-hadoop1.jar'
 
 # Complete URL for downloading the BigQuery Connector JAR file.
-BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/bigquery/bigquery-connector-0.6.1-SNAPSHOT-20150521-175606-hadoop1.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1463793407&Signature=MhWtS96xHADX51L4%2FusnpctWSRF2OYGyqrXuBSl0lSK8ET5hz9IpIGPcQZ02hSyL4JyOu0RvFkNxusXndUsr%2FkLWJTjDWPceT7DOXgxWi%2Fx9z5aO5UICIcIgrHfNyoMkeWv3NM8Sa39q1Wh0mqbPrMiM1lr3nT%2B%2BwylFDrsXQjI='
+BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/bigquery/bigquery-connector-0.7.0-hadoop1.jar'
 
 # Complete URL for downloading the configuration script.
 BDCONFIG='https://storage.googleapis.com/hadoop-tools/bdconfig/bdconfig-0.28.1.tar.gz'

diff --git a/extensions/spark/spark_env.sh b/extensions/spark/spark_env.sh
@@ -24,8 +24,8 @@ SPARK_MODE="standalone"
 
 # URIs of tarballs to install.
 SCALA_TARBALL_URI='gs://spark-dist/scala-2.10.3.tgz'
-SPARK_HADOOP1_TARBALL_URI='gs://spark-dist/spark-1.3.0-bin-hadoop1.tgz'
-SPARK_HADOOP2_TARBALL_URI='gs://spark-dist/spark-1.3.0-bin-hadoop2.4.tgz'
+SPARK_HADOOP1_TARBALL_URI='gs://spark-dist/spark-1.3.1-bin-hadoop1.tgz'
+SPARK_HADOOP2_TARBALL_URI='gs://spark-dist/spark-1.3.1-bin-hadoop2.6.tgz'
 
 # Directory on each VM in which to install each package.
 SCALA_INSTALL_DIR='/home/hadoop/scala-install'

diff --git a/hadoop2_env.sh b/hadoop2_env.sh
@@ -18,7 +18,7 @@ GCS_CACHE_CLEANER_LOGGER='INFO,RFA'
 
 # URI of Hadoop tarball to be deployed. Must begin with gs:// or http(s)://
 # Use 'gsutil ls gs://hadoop-dist/hadoop-*.tar.gz' to list Google supplied options
-HADOOP_TARBALL_URI="gs://hadoop-dist/hadoop-2.5.2.tar.gz"
+HADOOP_TARBALL_URI="gs://hadoop-dist/hadoop-2.6.0.tar.gz"
 
 # Directory holding config files and scripts for Hadoop
 HADOOP_CONF_DIR="${HADOOP_INSTALL_DIR}/etc/hadoop"
@@ -44,9 +44,9 @@ CORES_PER_REDUCE_TASK=2.0
 CORES_PER_APP_MASTER=2.0
 
 # Connector with Hadoop AbstractFileSystem implemenation for YARN
-GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/gcs/gcs-connector-1.3.4-SNAPSHOT-hadoop2-20150521-171734.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1463791002&Signature=PdhJSkuW6dAyp8i%2BvXf32NuoYezTVseO3bN422Wr36WSFKpetThfgKuez9N1o%2BRbFbd5qSA3J0E5yQ6%2Bh9F3O86%2BIlUFDmJeg4NTNyKjzC1%2B5FoQLqQ0JTcdKtLXwTIO3ugDm1AP7A6to%2F0ivPf%2BVAiqiNhKLAvglGyfB5riUEk='
+GCS_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/gcs/gcs-connector-1.4.0-hadoop2.jar'
 
-BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib-dev/bigquery/bigquery-connector-0.6.1-SNAPSHOT-20150521-175606-hadoop2.jar?GoogleAccessId=359641935755-j2hkfvkvflpvguhuj2dajativ5ft8856@developer.gserviceaccount.com&Expires=1463793407&Signature=mVEcy8nvAitm4jUFURwCH1nleHEeFsueY1yCHmicZhvlvn3s%2BA1slGCDTy14UrWxLa0QmwEuCDMRGXfIE6ogVPmI8u7%2B6VyBg72z0zjeiPj5JgpiV424OB6uRkVjSl480O%2BOUg9QZhxHnIxORx12a2GuhOtpuXCjzUdDS6JOqCE='
+BIGQUERY_CONNECTOR_JAR='https://storage.googleapis.com/hadoop-lib/bigquery/bigquery-connector-0.7.0-hadoop2.jar'
 
 
 HDFS_DATA_DIRS_PERM='700'