Skip to content

Commit

Permalink
Merge pull request #1 from apache/master
Browse files Browse the repository at this point in the history 
Merge from Upstream - per Scott T request
  • Loading branch information
@pawelpodgorski
pawelpodgorski authored Aug 12, 2020
2 parents e3ff179 + 265f7f2 commit e82da11
Show file tree
Hide file tree
Showing 230 changed files with 21,033 additions and 16,039 deletions.
22 changes: 1 addition & 21 deletions agents-audit/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<parent>
<groupId>org.apache.ranger</groupId>
<artifactId>ranger</artifactId>
<version>2.1.0-SNAPSHOT</version>
<version>3.0.0-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<dependencies>
Expand Down Expand Up @@ -158,26 +158,6 @@
<artifactId>elasticsearch-secure-sm</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.plugin</groupId>
<artifactId>parent-join-client</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.plugin</groupId>
<artifactId>aggs-matrix-stats-client</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.plugin</groupId>
<artifactId>rank-eval-client</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.plugin</groupId>
<artifactId>lang-mustache-client</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore-nio</artifactId>
Expand Down
106 changes: 81 additions & 25 deletions ...udit/src/main/java/org/apache/ranger/audit/destination/ElasticSearchAuditDestination.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,33 +19,45 @@

package org.apache.ranger.audit.destination;

import java.io.File;
import java.security.PrivilegedActionException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.auth.AuthSchemeProvider;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.config.Lookup;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.impl.auth.SPNegoSchemeFactory;
import org.apache.ranger.audit.model.AuditEventBase;
import org.apache.ranger.audit.model.AuthzAuditEvent;
import org.apache.ranger.audit.provider.MiscUtil;
import org.apache.ranger.authorization.credutils.CredentialsProviderUtil;
import org.apache.ranger.authorization.credutils.kerberos.KerberosCredentialsProvider;
import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;

import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosTicket;

public class ElasticSearchAuditDestination extends AuditDestination {
private static final Log LOG = LogFactory.getLog(ElasticSearchAuditDestination.class);
Expand All @@ -57,7 +69,7 @@ public class ElasticSearchAuditDestination extends AuditDestination {
public static final String CONFIG_PROTOCOL = "protocol";
public static final String CONFIG_INDEX = "index";
public static final String CONFIG_PREFIX = "ranger.audit.elasticsearch";
public static final String DEFAULT_INDEX = "audit";
public static final String DEFAULT_INDEX = "ranger_audits";

private String index = "index";
private volatile RestHighLevelClient client = null;
Expand All @@ -66,6 +78,7 @@ public class ElasticSearchAuditDestination extends AuditDestination {
private int port;
private String password;
private String hosts;
private Subject subject;

public ElasticSearchAuditDestination() {
propPrefix = CONFIG_PREFIX;
Expand All @@ -86,7 +99,7 @@ public void init(Properties props, String propPrefix) {
}

private String connectionString() {
return String.format("%s://%s@%s:%s/%s", protocol, user, hosts, port, index);
return String.format(Locale.ROOT, "User:%s, %s://%s:%s/%s", user, protocol, hosts, port, index);
}

@Override
Expand Down Expand Up @@ -116,7 +129,7 @@ public boolean log(Collection<AuditEventBase> events) {
AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
String id = authzEvent.getEventId();
Map<String, Object> doc = toDoc(authzEvent);
bulkRequest.add(new IndexRequest(index).id(id).source(doc).type(""));
bulkRequest.add(new IndexRequest(index).id(id).source(doc));
}
} catch (Exception ex) {
addFailedCount(eventList.size());
Expand Down Expand Up @@ -172,38 +185,79 @@ synchronized RestHighLevelClient getClient() {
}
}
}
if (subject != null) {
KerberosTicket ticket = CredentialsProviderUtil.getTGT(subject);
try {
if (new Date().getTime() > ticket.getEndTime().getTime()){
client = null;
CredentialsProviderUtil.ticketExpireTime80 = 0;
newClient();
} else if (CredentialsProviderUtil.ticketWillExpire(ticket)) {
subject = CredentialsProviderUtil.login(user, password);
}
} catch (PrivilegedActionException e) {
LOG.error("PrivilegedActionException:", e);
throw new RuntimeException(e);
}
}
return client;
}

private final AtomicLong lastLoggedAt = new AtomicLong(0);

private RestHighLevelClient newClient() {
try {
final CredentialsProvider credentialsProvider;
if(!user.isEmpty()) {
credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(user, password));
public static RestClientBuilder getRestClientBuilder(String urls, String protocol, String user, String password, int port) {
RestClientBuilder restClientBuilder = RestClient.builder(
MiscUtil.toArray(urls, ",").stream()
.map(x -> new HttpHost(x, port, protocol))
.<HttpHost>toArray(i -> new HttpHost[i])
);
if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && !user.equalsIgnoreCase("NONE") && !password.equalsIgnoreCase("NONE")) {
if (password.contains("keytab") && new File(password).exists()) {
final KerberosCredentialsProvider credentialsProvider =
CredentialsProviderUtil.getKerberosCredentials(user, password);
Lookup<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
.register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory()).build();
restClientBuilder.setHttpClientConfigCallback(clientBuilder -> {
clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
clientBuilder.setDefaultAuthSchemeRegistry(authSchemeRegistry);
return clientBuilder;
});
} else {
credentialsProvider = null;
final CredentialsProvider credentialsProvider =
CredentialsProviderUtil.getBasicCredentials(user, password);
restClientBuilder.setHttpClientConfigCallback(clientBuilder ->
clientBuilder.setDefaultCredentialsProvider(credentialsProvider));
}
} else {
LOG.error("ElasticSearch Credentials not provided!!");
final CredentialsProvider credentialsProvider = null;
restClientBuilder.setHttpClientConfigCallback(clientBuilder ->
clientBuilder.setDefaultCredentialsProvider(credentialsProvider));
}
return restClientBuilder;
}

RestHighLevelClient restHighLevelClient = new RestHighLevelClient(
RestClient.builder(
MiscUtil.toArray(hosts, ",").stream()
.map(x -> new HttpHost(x, port, protocol))
.<HttpHost>toArray(i -> new HttpHost[i])
).setHttpClientConfigCallback(clientBuilder ->
(credentialsProvider != null) ? clientBuilder.setDefaultCredentialsProvider(credentialsProvider) : clientBuilder));
LOG.debug("Initialized client");
private RestHighLevelClient newClient() {
try {
if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && password.contains("keytab") && new File(password).exists()) {
subject = CredentialsProviderUtil.login(user, password);
}
RestClientBuilder restClientBuilder =
getRestClientBuilder(hosts, protocol, user, password, port);
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(restClientBuilder);
if (LOG.isDebugEnabled()) {
LOG.debug("Initialized client");
}
boolean exits = false;
try {
exits = restHighLevelClient.indices().open(new OpenIndexRequest(this.index), RequestOptions.DEFAULT).isShardsAcknowledged();
} catch (Exception e) {
LOG.warn("Error validating index " + this.index);
}
if(exits) {
LOG.debug("Index exists");
if (LOG.isDebugEnabled()) {
LOG.debug("Index exists");
}
} else {
LOG.info("Index does not exist");
}
Expand All @@ -228,15 +282,17 @@ private String getHosts() {
if (urls != null) {
urls = urls.trim();
}
if (urls != null && urls.equalsIgnoreCase("NONE")) {
if ("NONE".equalsIgnoreCase(urls)) {
urls = null;
}
return urls;
}

private String getStringProperty(Properties props, String propName, String defaultValue) {
String value = MiscUtil.getStringProperty(props, propName);
if (null == value) return defaultValue;
if (null == value) {
return defaultValue;
}
return value;
}

Expand Down
20 changes: 1 addition & 19 deletions agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ public class AuditProviderFactory {
public static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled";
public static final String AUDIT_KAFKA_IS_ENABLED_PROP = "xasecure.audit.kafka.is.enabled";
public static final String AUDIT_SOLR_IS_ENABLED_PROP = "xasecure.audit.solr.is.enabled";
public static final String AUDIT_ELASTICSEARCH_IS_ENABLED_PROP = "xasecure.audit.elasticsearch.is.enabled";

public static final String AUDIT_DEST_BASE = "xasecure.audit.destination";
public static final String AUDIT_SHUTDOWN_HOOK_MAX_WAIT_SEC = "xasecure.audit.shutdown.hook.max.wait.seconds";
Expand Down Expand Up @@ -139,8 +138,6 @@ public synchronized void init(Properties props, String appType) {
AUDIT_KAFKA_IS_ENABLED_PROP, false);
boolean isAuditToSolrEnabled = MiscUtil.getBooleanProperty(props,
AUDIT_SOLR_IS_ENABLED_PROP, false);
boolean isAuditToElasticsearchEnabled = MiscUtil.getBooleanProperty(props,
AUDIT_ELASTICSEARCH_IS_ENABLED_PROP, false);

boolean isAuditFileCacheProviderEnabled = MiscUtil.getBooleanProperty(props,
AUDIT_IS_FILE_CACHE_PROVIDER_ENABLE_PROP, false);
Expand Down Expand Up @@ -283,8 +280,7 @@ public synchronized void init(Properties props, String appType) {
if (!isEnabled
|| !(isAuditToDbEnabled || isAuditToHdfsEnabled
|| isAuditToKafkaEnabled || isAuditToLog4jEnabled
|| isAuditToSolrEnabled || isAuditToElasticsearchEnabled
|| providers.size() == 0)) {
|| isAuditToSolrEnabled || providers.size() == 0)) {
LOG.info("AuditProviderFactory: Audit not enabled..");

mProvider = getDefaultProvider();
Expand Down Expand Up @@ -373,20 +369,6 @@ public synchronized void init(Properties props, String appType) {
}
}

if (isAuditToElasticsearchEnabled) {
LOG.info("ElasticsearchAuditProvider is enabled");
ElasticSearchAuditDestination elasticSearchAuditDestination = new ElasticSearchAuditDestination();
elasticSearchAuditDestination.init(props);

if (elasticSearchAuditDestination.isAsync()) {
AsyncAuditProvider asyncProvider = new AsyncAuditProvider(
"MyElasticSearchAuditProvider", 1000, 1000, elasticSearchAuditDestination);
providers.add(asyncProvider);
} else {
providers.add(elasticSearchAuditDestination);
}
}

if (isAuditToLog4jEnabled) {
Log4jAuditProvider log4jProvider = new Log4jAuditProvider();

Expand Down
4 changes: 2 additions & 2 deletions agents-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
<parent>
<groupId>org.apache.ranger</groupId>
<artifactId>ranger</artifactId>
<version>2.1.0-SNAPSHOT</version>
<version>3.0.0-SNAPSHOT</version>
<relativePath>..</relativePath>
</parent>
<dependencies>
Expand Down Expand Up @@ -108,7 +108,7 @@
<dependency>
<groupId>org.apache.ranger</groupId>
<artifactId>ranger-plugin-classloader</artifactId>
<version>2.1.0-SNAPSHOT</version>
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
</dependencies>
Expand Down
4 changes: 2 additions & 2 deletions ...ts-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,18 +457,18 @@ private RangerAccessResult zoneAwareAccessEvaluationWithNoAudit(RangerAccessRequ
policyRepository = policyEngine.getRepositoryForZone(zoneName);

ret = evaluatePoliciesNoAudit(request, policyType, zoneName, policyRepository, tagPolicyRepository);
ret.setZoneName(zoneName);

if (ret.getIsAllowed()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Zone:[" + zoneName + "] allowed access. Completed processing other zones");
}
ret.setZoneName(zoneName);
break;
}
}
}

if (request.isAccessTypeAny() && CollectionUtils.isEmpty(zoneNames) && ret != null && !ret.getIsAllowed() && MapUtils.isNotEmpty(policyEngine.getZonePolicyRepositories())) {
if (request.isAccessTypeAny() && (request.getResource() == null || CollectionUtils.isEmpty(request.getResource().getKeys())) && ret != null && !ret.getIsAllowed() && MapUtils.isNotEmpty(policyEngine.getZonePolicyRepositories())) {
// resource is empty and access is ANY
if (LOG.isDebugEnabled()) {
LOG.debug("Process all security-zones");
Expand Down
Loading

0 comments on commit e82da11

Please sign in to comment.