Skip to content

Commit

Permalink
RANGER-3569 : Support Ranger KMS integration with Google cloud HSM
Browse files Browse the repository at this point in the history
  • Loading branch information
@mateenmansoori
mateenmansoori authored and Dhaval Shah committed Jan 17, 2022
1 parent 20046b2 commit 1af1252
Show file tree
Hide file tree
Showing 14 changed files with 629 additions and 48 deletions.
39 changes: 37 additions & 2 deletions distro/src/main/assembly/kms.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@
<include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include>
<include>org.apache.httpcomponents:httpclient:jar:${kms.httpcomponents.httpclient.version}</include>
<include>org.noggit:noggit:jar:${noggit.version}</include>
<include>com.google.protobuf:protobuf-java:jar:${protobuf-java.version}</include>
<include>com.google.protobuf:protobuf-java:jar:${gcp.protobuf-java.version}</include>
<include>org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version}</include>
<include>org.apache.htrace:htrace-core4:jar:${htrace-core.version}</include>
<include>org.codehaus.woodstox:stax2-api</include>
Expand Down Expand Up @@ -160,6 +160,40 @@
<include>com.microsoft.azure:azure-mgmt-eventhub</include>
<include>com.microsoft.azure:azure-mgmt-eventhub</include>
<include>com.microsoft.azure:azure-keyvault-cryptography</include>
<!-- GCP -->
<include>com.google.cloud:google-cloud-kms</include>
<include>io.grpc:grpc-api</include>
<include>io.grpc:grpc-context</include>
<include>io.grpc:grpc-stub</include>
<include>io.grpc:grpc-protobuf</include>
<include>io.grpc:grpc-protobuf-lite</include>
<include>com.google.api:api-common</include>
<include>javax.annotation:javax.annotation-api</include>
<include>com.google.auto.value:auto-value-annotations</include>
<include>com.google.api.grpc:proto-google-common-protos</include>
<include>com.google.api.grpc:proto-google-cloud-kms-v1</include>
<include>com.google.api:gax</include>
<include>io.opencensus:opencensus-api</include>
<include>com.google.api:gax-grpc</include>
<include>com.google.auth:google-auth-library-credentials</include>
<include>io.grpc:grpc-netty-shaded</include>
<include>io.perfmark:perfmark-api</include>
<include>io.grpc:grpc-core</include>
<include>com.google.android:annotations</include>
<include>io.grpc:grpc-alts</include>
<include>io.grpc:grpc-grpclb</include>
<include>com.google.protobuf:protobuf-java-util</include>
<include>org.conscrypt:conscrypt-openjdk-uber</include>
<include>org.threeten:threetenbp</include>
<include>io.grpc:grpc-auth</include>
<include>com.google.api.grpc:proto-google-iam-v1</include>
<include>com.google.auth:google-auth-library-oauth2-http</include>
<include>com.google.http-client:google-http-client</include>
<include>io.opencensus:opencensus-contrib-http-util</include>
<include>com.google.http-client:google-http-client-gson</include>
<include>io.grpc:grpc-xds</include>
<include>io.grpc:grpc-services</include>
<include>io.opencensus:opencensus-proto</include>
</includes>
</binaries>
</moduleSet>
Expand Down Expand Up @@ -191,7 +225,7 @@
<include>org.apache.tomcat.embed:tomcat-embed*</include>
<include>org.apache.tomcat:tomcat-annotations-api*</include>
<include>org.eclipse.jdt.core.compiler:ecj:jar:P20140317-1600</include>
<include>com.google.protobuf:protobuf-java:jar:${protobuf-java.version}</include>
<include>com.google.protobuf:protobuf-java:jar:${gcp.protobuf-java.version}</include>
<include>org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version}</include>
<include>org.apache.htrace:htrace-core4:jar:${htrace-core.version}</include>
<include>org.apache.solr:solr-solrj:jar:${solr.version}</include>
Expand Down Expand Up @@ -392,6 +426,7 @@
<include>DBMKTOKEYSECURE.sh</include>
<include>DBMKTOAZUREKEYVAULT.sh</include>
<include>KEYSECUREMKTOKMSDB.sh</include>
<include>MigrateMKeyStorageDbToGCP.sh</include>
</includes>
<fileMode>544</fileMode>
</fileSet>
Expand Down
35 changes: 34 additions & 1 deletion kms/config/kms-webapp/dbks-site.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,40 @@
<description>Azure Key Vault url</description>
</property>
<!--Azure Key Vault END-->


<!-- Google Cloud KMS start -->
<property>
<name>ranger.kms.gcp.enabled</name>
<value>false</value>
<description>Flag for Google Cloud HSM e.g - true or false</description>
</property>
<property>
<name>ranger.kms.gcp.keyring.id</name>
<value></value>
<description>Name of KeyRing.</description>
</property>
<property>
<name>ranger.kms.gcp.cred.file</name>
<value></value>
<description>Absolute path of downloaded json credential file, e.g - /path/to/credFile.json</description>
</property>
<property>
<name>ranger.kms.gcp.project.id</name>
<value></value>
<description>Name of project on Google Cloud HSM.</description>
</property>
<property>
<name>ranger.kms.gcp.location.id</name>
<value></value>
<description>GCP KeyRing location id, e.g - us-east1, global etc.</description>
</property>
<property>
<name>ranger.kms.gcp.masterkey.name</name>
<value></value>
<description>GCP Master Key Name.</description>
</property>
<!-- Google Cloud KMS end -->

<!-- HSM Config -->
<property>
<name>ranger.ks.hsm.type</name>
Expand Down
14 changes: 14 additions & 0 deletions kms/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand Down Expand Up @@ -483,6 +487,16 @@
<artifactId>nimbus-jose-jwt</artifactId>
<version>${nimbus-jose-jwt.version}</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-kms</artifactId>
<version>${google.cloud.kms}</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${gcp.protobuf-java.version}</version>
</dependency>
</dependencies>
<build>
<pluginManagement>
Expand Down
46 changes: 46 additions & 0 deletions kms/scripts/MigrateMKeyStorageDbToGCP.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------------------
if [ -z "${JAVA_HOME}" ]; then
echo "PLEASE EXPORT VARAIBLE JAVA_HOME"
exit;
else
echo "JAVA_HOME : "$JAVA_HOME
fi

if [ -z "${RANGER_KMS_HOME}" ]; then
echo "PLEASE EXPORT VARAIBLE RANGER_KMS_HOME"
exit;
else
echo "RANGER_KMS_HOME : "$RANGER_KMS_HOME
fi

if [ -z "${RANGER_KMS_CONF}" ]; then
echo "PLEASE EXPORT VARAIBLE RANGER_KMS_CONF"
exit;
else
echo "RANGER_KMS_CONF : "$RANGER_KMS_CONF
fi

if [ -z "${SQL_CONNECTOR_JAR}" ]; then
echo "PLEASE EXPORT VARAIBLE SQL_CONNECTOR_JAR"
exit;
else
echo "SQL_CONNECTOR_JAR : "$SQL_CONNECTOR_JAR
fi

cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_CONF}:${RANGER_KMS_HOME}/ews/webapp/WEB-INF/classes/lib/*:${SQL_CONNECTOR_JAR}:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF:${RANGER_KMS_CONF}/*"
${JAVA_HOME}/bin/java -cp "${cp}" org.apache.hadoop.crypto.key.MigrateDBMKeyToGCP ${1} ${2} ${3} ${4} ${5}
8 changes: 7 additions & 1 deletion kms/scripts/install.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,14 @@ AZURE_MASTER_KEY_TYPE=RSA
ZONE_KEY_ENCRYPTION_ALGO=RSA_OAEP
AZURE_KEYVAULT_URL=https://shahkeyvault.vault.azure.net/

#------------------------- Ranger Google Cloud HSM ------------------------------
IS_GCP_ENABLED=false
GCP_KEYRING_ID=
GCP_CRED_JSON_FILE=/full/path/to/credfile.json
GCP_PROJECT_ID=
GCP_LOCATION_ID=
GCP_MASTER_KEY_NAME=MyMasterKeyNameChangeIt

#
# ------- UNIX User CONFIG ----------------
#
unix_user=kms
Expand Down
56 changes: 56 additions & 0 deletions kms/scripts/setup.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,13 @@ AZURE_MASTER_KEY_TYPE=$(get_prop 'AZURE_MASTER_KEY_TYPE' $PROPFILE)
ZONE_KEY_ENCRYPTION_ALGO=$(get_prop 'ZONE_KEY_ENCRYPTION_ALGO' $PROPFILE)
AZURE_KEYVAULT_URL=$(get_prop 'AZURE_KEYVAULT_URL' $PROPFILE)

IS_GCP_ENABLED=$(get_prop 'IS_GCP_ENABLED' $PROPFILE)
GCP_KEYRING_ID=$(get_prop 'GCP_KEYRING_ID' $PROPFILE)
GCP_CRED_JSON_FILE=$(get_prop 'GCP_CRED_JSON_FILE' $PROPFILE)
GCP_PROJECT_ID=$(get_prop 'GCP_PROJECT_ID' $PROPFILE)
GCP_LOCATION_ID=$(get_prop 'GCP_LOCATION_ID' $PROPFILE)
GCP_MASTER_KEY_NAME=$(get_prop 'GCP_MASTER_KEY_NAME' $PROPFILE)

kms_principal=$(get_prop 'kms_principal' $PROPFILE)
kms_keytab=$(get_prop 'kms_keytab' $PROPFILE)
hadoop_conf=$(get_prop 'hadoop_conf' $PROPFILE)
Expand Down Expand Up @@ -471,6 +478,14 @@ setup_kms(){
cd ${oldP}
}

checkIfEmpty() {
if [ -z "$1" ]
then
echo "Error - Since GCP is enabled, Please provide valid value for '$2', Found : '$1'";
exit 1
fi
}

update_properties() {
newPropertyValue=''
echo "export JAVA_HOME=${JAVA_HOME}" > ${WEBAPP_ROOT}/WEB-INF/classes/conf/java_home.sh
Expand Down Expand Up @@ -643,6 +658,7 @@ update_properties() {
HSM_ENABLED=`echo $HSM_ENABLED | tr '[:lower:]' '[:upper:]'`
KEYSECURE_ENABLED=`echo $KEYSECURE_ENABLED | tr '[:lower:]' '[:upper:]'`
AZURE_KEYVAULT_ENABLED=`echo $AZURE_KEYVAULT_ENABLED | tr '[:lower:]' '[:upper:]'`
IS_GCP_ENABLED=`echo $IS_GCP_ENABLED | tr '[:lower:]' '[:upper:]'`

if [ "${keystore}" != "" ]
then
Expand Down Expand Up @@ -888,6 +904,46 @@ update_properties() {

fi

########### RANGER GCP #################
if [ "${IS_GCP_ENABLED}" != "TRUE" ]
then
propertyName=ranger.kms.gcp.enabled
newPropertyValue="false"
updatePropertyToFilePy $propertyName $newPropertyValue $to_file
else
propertyName=ranger.kms.gcp.enabled
newPropertyValue="true"
updatePropertyToFilePy $propertyName $newPropertyValue $to_file

propertyName=ranger.kms.gcp.keyring.id
newPropertyValue="${GCP_KEYRING_ID}"
checkIfEmpty "$newPropertyValue" "GCP_KEYRING_ID"
updatePropertyToFilePy $propertyName $newPropertyValue $to_file

propertyName=ranger.kms.gcp.cred.file
newPropertyValue="${GCP_CRED_JSON_FILE}"
if [ "${newPropertyValue: -5}" != ".json" ]
then
echo "Error - GCP Credential file must be in a json format, Provided file : ${newPropertyValue}";
exit 1
fi
updatePropertyToFilePy $propertyName $newPropertyValue $to_file

propertyName=ranger.kms.gcp.project.id
newPropertyValue="${GCP_PROJECT_ID}"
checkIfEmpty "$newPropertyValue" "GCP_PROJECT_ID"
updatePropertyToFilePy $propertyName $newPropertyValue $to_file

propertyName=ranger.kms.gcp.location.id
newPropertyValue="${GCP_LOCATION_ID}"
checkIfEmpty "$newPropertyValue" "GCP_LOCATION_ID"
updatePropertyToFilePy $propertyName $newPropertyValue $to_file

propertyName=ranger.kms.gcp.masterkey.name
newPropertyValue="${GCP_MASTER_KEY_NAME}"
checkIfEmpty "$newPropertyValue" "GCP_MASTER_KEY_NAME"
updatePropertyToFilePy $propertyName $newPropertyValue $to_file
fi

to_file_kms_site=$PWD/ews/webapp/WEB-INF/classes/conf/ranger-kms-site.xml
if test -f $to_file_kms_site; then
Expand Down
Loading

0 comments on commit 1af1252

Please sign in to comment.