Merge branch 'apache:master' into RANGER-4342

apache · Sep 9, 2023 · 97cc094 · 97cc094
2 parents 3c79fe8 + 84cb3c4
commit 97cc094
Show file tree

Hide file tree

Showing 369 changed files with 5,867 additions and 2,933 deletions.
diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml
@@ -361,6 +361,28 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.apache.orc</groupId>
+            <artifactId>orc-shims</artifactId>
+            <version>${orc.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>io.airlift</groupId>
+            <artifactId>aircompressor</artifactId>
+            <version>${aircompressor.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-bundle</artifactId>

diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditWriterFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditWriterFactory.java
@@ -61,7 +61,7 @@ public void init(Properties props, String propPrefix, String auditProviderName,
         this.propPrefix        = propPrefix;
         this.auditProviderName = auditProviderName;
         this.auditConfigs      = auditConfigs;
-        String auditFileType   = MiscUtil.getStringProperty(props, propPrefix + ".filetype", AUDIT_FILETYPE_DEFAULT);
+        String auditFileType   = MiscUtil.getStringProperty(props, propPrefix + ".batch.filequeue.filetype", AUDIT_FILETYPE_DEFAULT);
         String writerClass     = MiscUtil.getStringProperty(props, propPrefix + ".filewriter.impl");
 
         auditWriter = StringUtils.isEmpty(writerClass) ? createWriter(getDefaultWriter(auditFileType)) : createWriter(writerClass);

diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
@@ -38,6 +38,11 @@ public abstract class BaseAuditHandler implements AuditHandler {
 
 	static final String AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP = "xasecure.audit.log.failure.report.min.interval.ms";
 
+	static final String  AUDIT_LOG_STATUS_LOG_ENABLED              = "xasecure.audit.log.status.log.enabled";
+	static final String  AUDIT_LOG_STATUS_LOG_INTERVAL_SEC         = "xasecure.audit.log.status.log.interval.sec";
+	static final boolean DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED      = false;
+	static final long    DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC = 5 * 60; // 5 minutes
+
 	public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE                  = "xasecure.policymgr.clientssl.keystore";
 	public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE             = "xasecure.policymgr.clientssl.keystore.type";
 	public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL       = "xasecure.policymgr.clientssl.keystore.credential.file";
@@ -90,8 +95,10 @@ public abstract class BaseAuditHandler implements AuditHandler {
 	long lastStashedCount = 0;
 	long lastDeferredCount = 0;
 
-	long lastStatusLogTime = System.currentTimeMillis();
-	long statusLogIntervalMS = 1 * 60 * 1000;
+	boolean statusLogEnabled    = DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED;
+	long    statusLogIntervalMS = DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC * 1000;
+	long    lastStatusLogTime   = System.currentTimeMillis();
+	long    nextStatusLogTime   = lastStatusLogTime + statusLogIntervalMS;
 
 	protected Properties props = null;
 	protected Map<String, String> configProps = new HashMap<String, String>();
@@ -138,6 +145,19 @@ public void init(Properties props, String basePropertyName) {
 		mLogFailureReportMinIntervalInMs = MiscUtil.getIntProperty(props,
 				AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP, 60 * 1000);
 
+		boolean globalStatusLogEnabled     = MiscUtil.getBooleanProperty(props, AUDIT_LOG_STATUS_LOG_ENABLED, DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED);
+		long    globalStatusLogIntervalSec = MiscUtil.getLongProperty(props, AUDIT_LOG_STATUS_LOG_INTERVAL_SEC, DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC);
+
+		statusLogEnabled    = MiscUtil.getBooleanProperty(props, basePropertyName + ".status.log.enabled", globalStatusLogEnabled);
+		statusLogIntervalMS = MiscUtil.getLongProperty(props, basePropertyName + ".status.log.interval.sec", globalStatusLogIntervalSec) * 1000;
+
+		nextStatusLogTime = lastStatusLogTime + statusLogIntervalMS;
+
+		LOG.info(AUDIT_LOG_STATUS_LOG_ENABLED + "=" + globalStatusLogEnabled);
+		LOG.info(AUDIT_LOG_STATUS_LOG_INTERVAL_SEC + "=" + globalStatusLogIntervalSec);
+		LOG.info(basePropertyName + ".status.log.enabled=" + statusLogEnabled);
+		LOG.info(basePropertyName + ".status.log.interval.sec=" + (statusLogIntervalMS / 1000));
+
 		String configPropsNamePrefix = propPrefix + "." + PROP_CONFIG + ".";
 		for (Object propNameObj : props.keySet()) {
 			String propName = propNameObj.toString();
@@ -275,19 +295,21 @@ public long getLastDeferredCount() {
 		return lastDeferredCount;
 	}
 
+	public boolean isStatusLogEnabled() { return statusLogEnabled; }
+
 	public void logStatusIfRequired() {
-		long currTime = System.currentTimeMillis();
-		if ((currTime - lastStatusLogTime) > statusLogIntervalMS) {
+		if (System.currentTimeMillis() > nextStatusLogTime) {
 			logStatus();
 		}
 	}
 
 	public void logStatus() {
 		try {
 			long currTime = System.currentTimeMillis();
-
 			long diffTime = currTime - lastStatusLogTime;
+
 			lastStatusLogTime = currTime;
+			nextStatusLogTime = currTime + statusLogIntervalMS;
 
 			long diffCount = totalCount - lastIntervalCount;
 			long diffSuccess = totalSuccessCount - lastIntervalSuccessCount;
@@ -306,7 +328,7 @@ public void logStatus() {
 			lastStashedCount = totalStashedCount;
 			lastDeferredCount = totalDeferredCount;
 
-			if (LOG.isDebugEnabled()) {
+			if (statusLogEnabled) {
 				String finalPath = "";
 				String tFinalPath = getFinalPath();
 				if (!getName().equals(tFinalPath)) {
@@ -336,7 +358,7 @@ public void logStatus() {
 						: "")
 						+ (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount)
 						: "");
-				LOG.debug(msg);
+				LOG.info(msg);
 			}
 		} catch (Throwable t) {
 			LOG.error("Error while printing stats. auditProvider=" + getName());

diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java
@@ -56,8 +56,13 @@ public AuditAsyncQueue(AuditHandler consumer) {
 	 */
 	@Override
 	public boolean log(AuditEventBase event) {
+		logStatusIfRequired();
+
+		addTotalCount(1);
+
 		// Add to the queue and return ASAP
 		if (queue.size() >= getMaxQueueSize()) {
+			addFailedCount(1);
 			return false;
 		}
 		queue.add(event);
@@ -134,6 +139,17 @@ public void run() {
 		}
 	}
 
+	@Override
+	public void logStatus() {
+		super.logStatus();
+
+		if (isStatusLogEnabled()) {
+			logger.info("AuditAsyncQueue.log(name={}): totalCount={}, currentQueueLength={}", getName(), getTotalCount(), queue.size());
+		}
+	}
+
+	public int size() { return queue.size(); }
+
 	public void runLogAudit() {
 		while (true) {
 			try {
@@ -150,6 +166,8 @@ public void runLogAudit() {
 					eventList.add(event);
 					queue.drainTo(eventList, MAX_DRAIN - 1);
 					consumer.log(eventList);
+
+					logStatusIfRequired();
 				}
 			} catch (InterruptedException e) {
 				logger.info("Caught exception in consumer thread. Shutdown might be in progress");

diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
@@ -56,8 +56,13 @@ public AuditBatchQueue(AuditHandler consumer) {
 	 */
 	@Override
 	public boolean log(AuditEventBase event) {
-		// Add to batchQueue. Block if full
-		queue.add(event);
+		try {
+			// Add to batchQueue. Block if full
+			queue.put(event);
+		} catch (InterruptedException ex) {
+			throw new RuntimeException(ex);
+		}
+
 		return true;
 	}
 

diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/utils/ORCFileUtil.java b/agents-audit/src/main/java/org/apache/ranger/audit/utils/ORCFileUtil.java
@@ -404,7 +404,7 @@ protected CompressionKind getORCCompression(String compression) {
             case "lzo":
                 ret = CompressionKind.LZO;
                 break;
-            case "zlip":
+            case "zlib":
                 ret = CompressionKind.ZLIB;
                 break;
             case "none":

diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/utils/RangerORCAuditWriter.java b/agents-audit/src/main/java/org/apache/ranger/audit/utils/RangerORCAuditWriter.java
@@ -54,7 +54,7 @@ public void init(Properties props, String propPrefix, String auditProviderName,
         if (logger.isDebugEnabled()) {
             logger.debug("==> RangerORCAuditWriter.init()");
         }
-        init(props,propPrefix);
+        init(props,propPrefix,auditProviderName);
         super.init(props, propPrefix, auditProviderName, auditConfigs);
         if (logger.isDebugEnabled()) {
             logger.debug("<== RangerORCAuditWriter.init()");
@@ -174,7 +174,7 @@ public Collection<AuthzAuditEvent> getAuthzAuditEvents(Collection<String> events
         return ret;
     }
 
-    public void init(Properties props, String propPrefix) {
+    public void init(Properties props, String propPrefix, String auditProviderName) {
         compression    = MiscUtil.getStringProperty(props, propPrefix + "." + fileType +".compression");
         orcBufferSize  = MiscUtil.getIntProperty(props, propPrefix + "." + fileType +".buffersize",defaultbufferSize);
         orcStripeSize  = MiscUtil.getLongProperty(props, propPrefix + "." + fileType +".stripesize",defaultStripeSize);

diff --git a/...ommon/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerPluginConfig.java b/...ommon/src/main/java/org/apache/ranger/authorization/hadoop/config/RangerPluginConfig.java
@@ -48,6 +48,10 @@ public class RangerPluginConfig extends RangerConfiguration {
     private final boolean                   useForwardedIPAddress;
     private final String[]                  trustedProxyAddresses;
     private final String                    propertyPrefix;
+    private final boolean                   useRangerGroups;
+    private final boolean                   useOnlyRangerGroups;
+    private final boolean                   convertEmailToUsername;
+    private final boolean                   enableImplicitUserStoreEnricher;
     private       boolean                   isFallbackSupported;
     private       Set<String>               auditExcludedUsers  = Collections.emptySet();
     private       Set<String>               auditExcludedGroups = Collections.emptySet();
@@ -116,6 +120,11 @@ public RangerPluginConfig(String serviceType, String serviceName, String appId,
 
         this.policyEngineOptions = policyEngineOptions;
 
+        useRangerGroups                 = this.getBoolean(propertyPrefix + ".use.rangerGroups", false);
+        useOnlyRangerGroups             = this.getBoolean(propertyPrefix + ".use.only.rangerGroups", false);
+        convertEmailToUsername          = this.getBoolean(propertyPrefix + ".convert.emailToUser", false);
+        enableImplicitUserStoreEnricher = useRangerGroups || convertEmailToUsername || this.getBoolean(propertyPrefix + ".enable.implicit.userstore.enricher", false);
+
         LOG.info("" + policyEngineOptions);
     }
 
@@ -135,6 +144,10 @@ protected RangerPluginConfig(String serviceType, String serviceName, String appI
 
         this.policyEngineOptions = sourcePluginConfig.getPolicyEngineOptions();
 
+        this.useRangerGroups                 = sourcePluginConfig.useRangerGroups;
+        this.useOnlyRangerGroups             = sourcePluginConfig.useOnlyRangerGroups;
+        this.convertEmailToUsername          = sourcePluginConfig.convertEmailToUsername;
+        this.enableImplicitUserStoreEnricher = sourcePluginConfig.enableImplicitUserStoreEnricher;
     }
 
     public String getServiceType() {
@@ -169,6 +182,22 @@ public String getPropertyPrefix() {
         return propertyPrefix;
     }
 
+    public boolean isUseRangerGroups() {
+        return useRangerGroups;
+    }
+
+    public boolean isUseOnlyRangerGroups() {
+        return useOnlyRangerGroups;
+    }
+
+    public boolean isConvertEmailToUsername() {
+        return convertEmailToUsername;
+    }
+
+    public boolean isEnableImplicitUserStoreEnricher() {
+        return enableImplicitUserStoreEnricher;
+    }
+
     public boolean getIsFallbackSupported() {
         return isFallbackSupported;
     }

diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagForEval.java
@@ -33,9 +33,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -46,8 +43,6 @@
 @JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY)
 @JsonSerialize(include=JsonSerialize.Inclusion.NON_EMPTY)
 @JsonIgnoreProperties(ignoreUnknown=true)
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
 
 // This class needs above annotations for policy-engine unit tests involving RangerTagForEval objects that are initialized
 // from JSON specification

diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java b/agents-common/src/main/java/org/apache/ranger/plugin/errors/ValidationErrorCode.java
@@ -104,6 +104,9 @@ public enum ValidationErrorCode {
     POLICY_VALIDATION_ERR_NONEXISTANT_ZONE_NAME(3033, "Non-existent Zone name={0} in policy create"),
     POLICY_VALIDATION_ERR_SERVICE_NOT_ASSOCIATED_TO_ZONE(3048, "Service name = {0} is not associated to Zone name = {1}"),
     POLICY_VALIDATION_ERR_UNSUPPORTED_POLICY_ITEM_TYPE(3049, "Deny or deny-exceptions are not supported if policy has isDenyAllElse flag set to true"),
+    POLICY_VALIDATION_ERR_NULL_POLICY_ITEM_USER(3053, "policy items user was null"),
+    POLICY_VALIDATION_ERR_NULL_POLICY_ITEM_GROUP(3054, "policy items group was null"),
+    POLICY_VALIDATION_ERR_NULL_POLICY_ITEM_ROLE(3055, "policy items role was null"),
     POLICY_VALIDATION_ERR_INVALID_SERVICE_TYPE(4009," Invalid service type [{0}] provided for service [{1}]"),
 
     // SECURITY_ZONE Validations

diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/AuditFilter.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/AuditFilter.java
@@ -24,9 +24,6 @@
 import java.util.List;
 import java.util.Map;
 
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
 import org.codehaus.jackson.annotate.JsonAutoDetect;
 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
 import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility;
@@ -35,8 +32,6 @@
 @JsonAutoDetect(fieldVisibility=Visibility.ANY)
 @JsonSerialize(include=JsonSerialize.Inclusion.NON_EMPTY)
 @JsonIgnoreProperties(ignoreUnknown=true)
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
 public class AuditFilter {
     public enum AccessResult { DENIED, ALLOWED, NOT_DETERMINED }
 

diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/GroupInfo.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/GroupInfo.java
@@ -24,17 +24,12 @@
 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
 import org.codehaus.jackson.map.annotate.JsonSerialize;
 
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
 import java.util.HashMap;
 import java.util.Map;
 
 @JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY)
 @JsonSerialize(include=JsonSerialize.Inclusion.NON_EMPTY)
 @JsonIgnoreProperties(ignoreUnknown=true)
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
 public class GroupInfo extends RangerBaseModelObject implements java.io.Serializable {
 
     private static final long serialVersionUID = 1L;

diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerBaseModelObject.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerBaseModelObject.java
@@ -22,10 +22,6 @@
 import java.util.Date;
 import java.util.Map;
 
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-
 import org.apache.ranger.authorization.utils.StringUtil;
 import org.codehaus.jackson.annotate.JsonAutoDetect;
 import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility;
@@ -35,8 +31,6 @@
 @JsonAutoDetect(getterVisibility=Visibility.NONE, setterVisibility=Visibility.NONE, fieldVisibility=Visibility.ANY)
 @JsonSerialize(include=JsonSerialize.Inclusion.NON_EMPTY )
 @JsonIgnoreProperties(ignoreUnknown=true)
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
 public class RangerBaseModelObject implements java.io.Serializable {
 	private static final long serialVersionUID = 1L;
 

diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerMetrics.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerMetrics.java
@@ -21,10 +21,6 @@
 
 import java.util.Map;
 
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-
 import org.codehaus.jackson.annotate.JsonAutoDetect;
 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
 import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility;
@@ -33,8 +29,6 @@
 @JsonAutoDetect(fieldVisibility=Visibility.ANY)
 @JsonSerialize(include=JsonSerialize.Inclusion.NON_EMPTY)
 @JsonIgnoreProperties(ignoreUnknown=true)
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
 public class RangerMetrics {
 
     private Map<String, Object> data;

diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPluginInfo.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPluginInfo.java
@@ -25,9 +25,6 @@
 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
 import org.codehaus.jackson.map.annotate.JsonSerialize;
 
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
 import java.io.Serializable;
 import java.util.Date;
 import java.util.HashMap;
@@ -36,8 +33,6 @@
 @JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY)
 @JsonSerialize(include=JsonSerialize.Inclusion.NON_EMPTY)
 @JsonIgnoreProperties(ignoreUnknown=true)
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
 public class RangerPluginInfo implements Serializable {
 	private static final long serialVersionUID = 1L;