feat(kms): modifications for ranger kms

roles/ranger/common/templates/kms_install.properties.j2

@@ -14,14 +14,14 @@
 # limitations under the License.
 
 #
-# This file provides a list of the deployment variables for the Ranger KMS Web Application
+# This file provides a list of the deployment variables for the Ranger KMS Web Application 
 #
 
 #------------------------- DB CONFIG - BEGIN ----------------------------------
 # Uncomment the below if the DBA steps need to be run separately
 setup_mode={{ kms_install_properties.setup_mode }}
 
-PYTHON_COMMAND_INVOKER=python2
+PYTHON_COMMAND_INVOKER=python3
 
 #DB_FLAVOR=MYSQL|ORACLE|POSTGRES|MSSQL|SQLA
 DB_FLAVOR={{ kms_install_properties.DB_FLAVOR }}
@@ -40,8 +40,8 @@ SQL_CONNECTOR_JAR={{ kms_install_properties.SQL_CONNECTOR_JAR }}
 #
 # DB password for the DB admin user-id
 # **************************************************************************
-# ** If the password is left empty or not-defined here,
-# ** it will be prompted to enter the password during installation process
+# ** If the password is left empty or not-defined here, 
+# ** it will be prompted to enter the password during installation process 
 # **************************************************************************
 #
 #db_root_user=root|SYS|postgres|sa|dba
@@ -52,6 +52,7 @@ SQL_CONNECTOR_JAR={{ kms_install_properties.SQL_CONNECTOR_JAR }}
 db_root_user=root
 db_root_password=
 db_host={{ kms_install_properties.db_host }}
+#SSL config
 db_ssl_enabled=false
 db_ssl_required=false
 db_ssl_verifyServerCertificate=false
@@ -61,20 +62,35 @@ javax_net_ssl_keyStore=
 javax_net_ssl_keyStorePassword=
 javax_net_ssl_trustStore=
 javax_net_ssl_trustStorePassword=
+javax_net_ssl_trustStore_type=jks
+javax_net_ssl_keyStore_type=jks
+
+# For postgresql db
+db_ssl_certificate_file=
+
 #
 # DB UserId used for the Ranger KMS schema
 #
 db_name={{ kms_install_properties.db_name }}
 db_user={{ kms_install_properties.db_user }}
 db_password={{ kms_install_properties.db_password }}
 
+#For over-riding the jdbc url.
+is_override_db_connection_string=false
+db_override_connection_string=
+
 #------------------------- DB CONFIG - END ----------------------------------
 #KMS Server config
 ranger_kms_http_enabled=false
 ranger_kms_https_keystore_file={{ ranger_keystore_location }}
 ranger_kms_https_keystore_keyalias={{ ansible_fqdn }}
 ranger_kms_https_keystore_password={{ ranger_keystore_password }}
 
+#------------------------- RANGER KMS Install Dir ------------------
+realScriptPath=`readlink -f $0`
+realScriptDir=`dirname $realScriptPath`
+COMPONENT_INSTALL_DIR_NAME=`(cd $realScriptDir; pwd)`
+
 #------------------------- RANGER KMS Master Key Crypt Key ------------------
 KMS_MASTER_KEY_PASSWD={{ ranger_keyadmin_password }}
 
@@ -99,7 +115,36 @@ KEYSECURE_HOSTNAME=SunPKCS11-keysecurehn
 KEYSECURE_MASTER_KEY_SIZE=256
 KEYSECURE_LIB_CONFIG_PATH=/opt/safenetConf/64/8.3.1/sunpkcs11.cfg
 
-#
+#------------------------- Ranger Azure Key Vault ------------------------------
+AZURE_KEYVAULT_ENABLED=false
+AZURE_KEYVAULT_SSL_ENABLED=false
+AZURE_CLIENT_ID=50fd7ca6-fd4f-4785-a13f-1a6cc4e95e42
+AZURE_CLIENT_SECRET=<AzureKeyVaultPassword>
+AZURE_AUTH_KEYVAULT_CERTIFICATE_PATH=/home/machine/Desktop/azureAuthCertificate/keyvault-MyCert.pfx
+# Initialize below prop if your certificate file has any password
+#AZURE_AUTH_KEYVAULT_CERTIFICATE_PASSWORD=certPass
+AZURE_MASTERKEY_NAME=RangerMasterKey
+# E.G. RSA, RSA_HSM, EC, EC_HSM, OCT
+AZURE_MASTER_KEY_TYPE=RSA
+# E.G. RSA_OAEP, RSA_OAEP_256, RSA1_5, RSA_OAEP 
+ZONE_KEY_ENCRYPTION_ALGO=RSA_OAEP
+AZURE_KEYVAULT_URL=https://shahkeyvault.vault.azure.net/
+
+#------------------------- Ranger Google Cloud HSM ------------------------------
+IS_GCP_ENABLED=false
+GCP_KEYRING_ID=
+GCP_CRED_JSON_FILE=/full/path/to/credfile.json
+GCP_PROJECT_ID=
+GCP_LOCATION_ID=
+GCP_MASTER_KEY_NAME=MyMasterKeyNameChangeIt
+
+#------------------------- Ranger Tencent KMS ------------------------------
+TENCENT_KMS_ENABLED=false
+TENCENT_MASTERKEY_ID=
+TENCENT_CLIENT_ID=
+TENCENT_CLIENT_SECRET=
+TENCENT_CLIENT_REGION=
+
 # ------- UNIX User CONFIG ----------------
 #
 unix_user={{ ranger_kms_user }}
@@ -147,6 +192,20 @@ XAAUDIT.SOLR.PASSWORD=NONE
 XAAUDIT.SOLR.ZOOKEEPER=NONE
 XAAUDIT.SOLR.FILE_SPOOL_DIR={{ ranger_log_dir }}/kms/audit/solr/spool
 
+# Enable audit logs to ElasticSearch
+#Example
+#XAAUDIT.ELASTICSEARCH.ENABLE=true
+#XAAUDIT.ELASTICSEARCH.URL=localhost
+#XAAUDIT.ELASTICSEARCH.INDEX=audit
+
+XAAUDIT.ELASTICSEARCH.ENABLE=false
+XAAUDIT.ELASTICSEARCH.URL=NONE
+XAAUDIT.ELASTICSEARCH.USER=NONE
+XAAUDIT.ELASTICSEARCH.PASSWORD=NONE
+XAAUDIT.ELASTICSEARCH.INDEX=NONE
+XAAUDIT.ELASTICSEARCH.PORT=NONE
+XAAUDIT.ELASTICSEARCH.PROTOCOL=NONE
+
 # Enable audit logs to HDFS
 #Example
 #XAAUDIT.HDFS.ENABLE=true
@@ -168,6 +227,27 @@ XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY
 XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER
 XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER
 
+#Log4j Audit Provider
+XAAUDIT.LOG4J.ENABLE=true
+XAAUDIT.LOG4J.IS_ASYNC=false
+XAAUDIT.LOG4J.ASYNC.MAX.QUEUE.SIZE=10240
+XAAUDIT.LOG4J.ASYNC.MAX.FLUSH.INTERVAL.MS=30000
+XAAUDIT.LOG4J.DESTINATION.LOG4J=true
+XAAUDIT.LOG4J.DESTINATION.LOG4J.LOGGER=xaaudit
+
+# Enable audit logs to Amazon CloudWatch Logs
+#Example
+#XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=true
+#XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=ranger_audits
+#XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM={instance_id}
+#XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=/var/log/hive/audit/amazon_cloudwatch/spool
+
+XAAUDIT.AMAZON_CLOUDWATCH.ENABLE=false
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_GROUP=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.LOG_STREAM_PREFIX=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.FILE_SPOOL_DIR=NONE
+XAAUDIT.AMAZON_CLOUDWATCH.REGION=NONE
+
 # End of V3 properties
 
 
@@ -202,7 +282,7 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
 
-#Solr Audit Provder
+#Solr Audit Provider
 XAAUDIT.SOLR.IS_ENABLED=false
 XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
 XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000

roles/ranger/common/templates/ranger-kms.j2

@@ -25,15 +25,20 @@ fi
 action=$1
 arg2=$2
 arg3=$3
+
+if [ -z "${arg3}" ]
+then
+	arg3="hsmenabled"
+fi
+
 action=`echo $action | tr '[:lower:]' '[:upper:]'`
 realScriptPath=`readlink -f $0`
 realScriptDir=`dirname $realScriptPath`
 RANGER_KMS_DIR=`(cd $realScriptDir; pwd)`
 RANGER_KMS_EWS_DIR=${RANGER_KMS_DIR}/ews
 RANGER_KMS_EWS_CONF_DIR="${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/classes/conf"
-RANGER_KMS_EWS_LIB_DIR="${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/classes/lib"
 
-ranger_kms_max_heap_size={{ ranger_kms_heapsize }}
+ranger_kms_max_heap_size=1g
 
 if [ -f ${RANGER_KMS_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh ]; then
         . ${RANGER_KMS_DIR}/ews/webapp/WEB-INF/classes/conf/java_home.sh
@@ -45,9 +50,7 @@ for custom_env_script in `find ${RANGER_KMS_DIR}/ews/webapp/WEB-INF/classes/conf
         fi
 done
 
-JMX_OPTS=" {{ jmx_common_opts }} {{ jmx_exporter_kms_opts }} "
-RANGER_LOGS_OPTS="-Dlog4j.configuration=file:{{ ranger_kms_conf_dir }}/conf/log4j.properties -Dranger.log.dir={{ ranger_log_dir }} -Dranger.log.file={{ ranger_kms_log_file }} -Dranger.root.logger={{ ranger_root_logger_level }},{{ ranger_root_logger }}"
-JAVA_OPTS="${RANGER_LOGS_OPTS} ${JAVA_OPTS} -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=256m -Xmx${ranger_kms_max_heap_size}"
+JAVA_OPTS=" ${JAVA_OPTS}  -XX:MetaspaceSize=100m -XX:MaxMetaspaceSize=256m -Xmx${ranger_kms_max_heap_size} -Xms1g "
 
 if [ "$JAVA_HOME" != "" ]; then
         export PATH=$JAVA_HOME/bin:$PATH
@@ -98,8 +101,9 @@ KMS_CONFIG_FILENAME=ranger-kms-site.xml
 
 TOMCAT_LOG_DIR=${RANGER_KMS_LOG_DIR}
 
-TOMCAT_LOG_FILE=${TOMCAT_LOG_DIR}/catalina-ranger-kms.out
-TOMCAT_STOP_LOG_FILE=${TOMCAT_LOG_DIR}/catalina-ranger-kms.out
+TOMCAT_LOG_FILE=${TOMCAT_LOG_DIR}/catalina.out
+TOMCAT_STOP_LOG_FILE=${TOMCAT_LOG_DIR}/stop_catalina.out
+KMS_LOG_PROPERTIES_FILE=${RANGER_KMS_EWS_CONF_DIR}/kms-logback.xml
 
 if [ ! -d ${TOMCAT_LOG_DIR} ]
 then
@@ -108,10 +112,11 @@ fi
 
 KMS_CONF_DIR=${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/classes/conf
 SERVER_NAME=rangerkms
-JAVA_OPTS="${JAVA_OPTS} ${DB_SSL_PARAM} -Duser=${USER} -Dhostname=${HOSTNAME} -Dservername=${SERVER_NAME} -Dcatalina.base=${RANGER_KMS_EWS_DIR} -Dkms.config.dir=${KMS_CONF_DIR} -Dkms.log.dir=${TOMCAT_LOG_DIR} -cp ${RANGER_KMS_EWS_CONF_DIR}:${RANGER_KMS_EWS_LIB_DIR}/*:${RANGER_KMS_EWS_DIR}/webapp/lib/*:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH "
+cp="-cp ${RANGER_KMS_EWS_CONF_DIR}:${RANGER_KMS_EWS_DIR}/lib/*:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH"
+JAVA_OPTS="${JAVA_OPTS} ${DB_SSL_PARAM} -Dmetric.type=${arg3} -Duser=${USER} -Dhostname=${HOSTNAME} -Dservername=${SERVER_NAME} -Dcatalina.base=${RANGER_KMS_EWS_DIR} -Dkms.config.dir=${KMS_CONF_DIR} -Dlogback.configurationFile=file:${KMS_LOG_PROPERTIES_FILE} -Dkms.log.dir=${TOMCAT_LOG_DIR} $cp"
 createRangerKMSPid () {
 	SLEEP_TIME_AFTER_START=5
-	nohup java -D${PROC_NAME} ${JAVA_OPTS} ${JMX_OPTS} ${START_CLASS_NAME} ${KMS_CONFIG_FILENAME} > ${TOMCAT_LOG_FILE} 2>&1 &
+	nohup java -D${PROC_NAME} ${JAVA_OPTS} ${START_CLASS_NAME} ${KMS_CONFIG_FILENAME} > ${TOMCAT_LOG_FILE} 2>&1 &
 	VALUE_OF_PID=$!
 	echo "Starting Apache Ranger KMS Service"
 	sleep $SLEEP_TIME_AFTER_START
@@ -193,7 +198,7 @@ elif [ "${action}" == "METRIC" ]; then
 	metric;
 	exit
 elif [ "${action}" == "VERSION" ]; then
-	( cd ${RANGER_KMS_LIB_DIR} ; java -cp ranger-util-*.jar org.apache.ranger.common.RangerVersionInfo )
+	( cd ${RANGER_KMS_EWS_DIR}/webapp/WEB-INF/lib ; java -cp ranger-util-*.jar org.apache.ranger.common.RangerVersionInfo )
 	exit
 else
         echo "Invalid argument [$1];"

tdp_vars_defaults/ranger/ranger.yml

@@ -29,7 +29,7 @@ ranger_kms_conf_dir: "/etc/kms"
 # Ranger pid directories
 ranger_pid_dir: /var/run/ranger
 ranger_usersync_pid_dir: /var/run/ranger-usersync
-ranger_kms_pid_dir: /var/run/ranger-kms
+ranger_kms_pid_dir: /var/run/ranger_kms
 
 # Ranger logging configuration
 # Root logger should be: [RFA | DRFA]