From 7cac1d33090951e2f9dc62b51dc554d84083a03d Mon Sep 17 00:00:00 2001 From: Pradeep AgrawaL Date: Thu, 6 Jul 2023 12:38:56 +0530 Subject: [PATCH 01/46] RANGER-4308: Upgrade netty to 4.1.94-final --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9fc9467311..518202b90f 100644 --- a/pom.xml +++ b/pom.xml @@ -177,7 +177,7 @@ 3.0.0 1.10.19 5.1.49 - 4.1.85.Final + 4.1.94.Final 0.8 1.6.7 20211018.2 From bce5150e271c2e4da095b316fa5536da334d121e Mon Sep 17 00:00:00 2001 From: RakeshGuptaDev Date: Wed, 21 Jun 2023 13:09:28 +0530 Subject: [PATCH 02/46] RANGER-4025: Ranger improvement - Roles Import/export API for ranger admin Signed-off-by: Dineshkumar Yadav --- .../ranger/plugin/model/RangerRole.java | 56 +++ pom.xml | 1 + .../org/apache/ranger/biz/RoleDBStore.java | 10 +- .../org/apache/ranger/biz/RoleRefUpdater.java | 30 +- .../org/apache/ranger/biz/ServiceDBStore.java | 66 ++- .../java/org/apache/ranger/rest/RoleREST.java | 308 +++++++++++++- .../org/apache/ranger/rest/ServiceREST.java | 21 +- .../context/RangerPreAuthSecurityHandler.java | 11 + .../ranger/view/RangerExportRoleList.java | 48 +++ .../apache/ranger/view/RangerPolicyList.java | 4 + .../apache/ranger/view/RangerRoleList.java | 4 + .../org/apache/ranger/rest/TestRoleREST.java | 399 ++++++++++++++++++ .../apache/ranger/rest/TestServiceREST.java | 3 +- .../importRole/import_role_test_file.json | 120 ++++++ 14 files changed, 1040 insertions(+), 41 deletions(-) create mode 100644 security-admin/src/main/java/org/apache/ranger/view/RangerExportRoleList.java create mode 100644 security-admin/src/test/java/org/apache/ranger/rest/importRole/import_role_test_file.json diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerRole.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerRole.java index 682bbd6406..c7b3699df8 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerRole.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerRole.java @@ -195,4 +195,60 @@ private String getPrintableOptions(Map options) { return ret.toString(); } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + ((description == null) ? 0 : description.hashCode()); + result = prime * result + ((options == null) ? 0 : options.hashCode()); + result = prime * result + ((users == null) ? 0 : users.hashCode()); + result = prime * result + ((groups == null) ? 0 : groups.hashCode()); + result = prime * result + ((roles == null) ? 0 : roles.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerRole other = (RangerRole) obj; + if (name == null) { + if (other.name != null) { + return false; + } + } else if (other.name == null || !name.equals(other.name)) { + return false; + } + if (description == null) { + if (other.description != null) + return false; + } else if (!description.equals(other.description)) + return false; + if (options == null) { + if (other.options != null) + return false; + } else if (!options.equals(other.options)) + return false; + if (users == null) { + if (other.users != null) + return false; + } else if (!users.equals(other.users)) + return false; + if (groups == null) { + if (other.groups != null) + return false; + } else if (!groups.equals(other.groups)) + return false; + if (roles == null) { + if (other.roles != null) + return false; + } else if (!roles.equals(other.roles)) + return false; + return true; + } } diff --git a/pom.xml b/pom.xml index 518202b90f..d4f6262026 100644 --- a/pom.xml +++ b/pom.xml @@ -1200,6 +1200,7 @@ **/docs/src/site/resources/ranger-logo.svg **/docs/src/site/resources/override-banner.js **/docs/src/site/resources/smooth-scroll.js + **/importRole/*.json diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RoleDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/RoleDBStore.java index 12983688b9..1b156a2832 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RoleDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RoleDBStore.java @@ -88,7 +88,7 @@ public class RoleDBStore implements RoleStore { RangerAdminConfig config; - private Boolean populateExistingBaseFields = true; + private Boolean populateExistingBaseFields = false; AbstractPredicateUtil predicateUtil = null; @@ -111,7 +111,7 @@ public void initStore() { } @Override - public RangerRole createRole(RangerRole role, Boolean createNonExistUserGroup) throws Exception { + public RangerRole createRole(RangerRole role, Boolean createNonExistUserGroupRole) throws Exception { if (LOG.isDebugEnabled()) { LOG.debug("==> RoleDBStore.createRole()"); } @@ -131,7 +131,7 @@ public RangerRole createRole(RangerRole role, Boolean createNonExistUserGroup) t throw new Exception("Cannot create role:[" + role + "]"); } - roleRefUpdater.createNewRoleMappingForRefTable(createdRole, createNonExistUserGroup); + roleRefUpdater.createNewRoleMappingForRefTable(createdRole, createNonExistUserGroupRole); List trxLogList = roleService.getTransactionLog(createdRole, null, "create"); bizUtil.createTrxLog(trxLogList); @@ -139,7 +139,7 @@ public RangerRole createRole(RangerRole role, Boolean createNonExistUserGroup) t } @Override - public RangerRole updateRole(RangerRole role, Boolean createNonExistUserGroup) throws Exception { + public RangerRole updateRole(RangerRole role, Boolean createNonExistUserGroupRole) throws Exception { XXRole xxRole = daoMgr.getXXRole().findByRoleId(role.getId()); if (xxRole == null) { throw restErrorUtil.createRESTException("role with id: " + role.getId() + " does not exist"); @@ -160,7 +160,7 @@ public RangerRole updateRole(RangerRole role, Boolean createNonExistUserGroup) t throw new Exception("Cannot update role:[" + role + "]"); } - roleRefUpdater.createNewRoleMappingForRefTable(updatedRole, createNonExistUserGroup); + roleRefUpdater.createNewRoleMappingForRefTable(updatedRole, createNonExistUserGroupRole); roleService.updatePolicyVersions(updatedRole.getId()); diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RoleRefUpdater.java b/security-admin/src/main/java/org/apache/ranger/biz/RoleRefUpdater.java index 421b2312da..9e2f28716c 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RoleRefUpdater.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RoleRefUpdater.java @@ -72,12 +72,15 @@ public class RoleRefUpdater { @Autowired RangerTransactionSynchronizationAdapter rangerTransactionSynchronizationAdapter; + @Autowired + RoleDBStore roleStore; + @Autowired RangerBizUtil xaBizUtil; public RangerDaoManager getRangerDaoManager() { return daoMgr; } - public void createNewRoleMappingForRefTable(RangerRole rangerRole, Boolean createNonExistUserGroup) { + public void createNewRoleMappingForRefTable(RangerRole rangerRole, Boolean createNonExistUserGroupRole) { if (rangerRole == null) { return; } @@ -99,7 +102,7 @@ public void createNewRoleMappingForRefTable(RangerRole rangerRole, Boolean creat roleRoles.add(role.getName()); } - final boolean isCreateNonExistentUGs = createNonExistUserGroup && xaBizUtil.checkAdminAccess(); + final boolean isCreateNonExistentUGRs = createNonExistUserGroupRole && xaBizUtil.checkAdminAccess(); if (CollectionUtils.isNotEmpty(roleUsers)) { for (String roleUser : roleUsers) { @@ -110,7 +113,7 @@ public void createNewRoleMappingForRefTable(RangerRole rangerRole, Boolean creat RolePrincipalAssociator associator = new RolePrincipalAssociator(PolicyRefUpdater.PRINCIPAL_TYPE.USER, roleUser, roleId); if (!associator.doAssociate(false)) { - if (isCreateNonExistentUGs) { + if (isCreateNonExistentUGRs) { rangerTransactionSynchronizationAdapter.executeOnTransactionCommit(associator); } else { throw restErrorUtil.createRESTException("user with name: " + roleUser + " does not exist ", MessageEnums.INVALID_INPUT_DATA); @@ -128,7 +131,7 @@ public void createNewRoleMappingForRefTable(RangerRole rangerRole, Boolean creat RolePrincipalAssociator associator = new RolePrincipalAssociator(PolicyRefUpdater.PRINCIPAL_TYPE.GROUP, roleGroup, roleId); if (!associator.doAssociate(false)) { - if (isCreateNonExistentUGs) { + if (isCreateNonExistentUGRs) { rangerTransactionSynchronizationAdapter.executeOnTransactionCommit(associator); } else { throw restErrorUtil.createRESTException("Group with name: " + roleGroup + " does not exist ", MessageEnums.INVALID_INPUT_DATA); @@ -147,7 +150,11 @@ public void createNewRoleMappingForRefTable(RangerRole rangerRole, Boolean creat RolePrincipalAssociator associator = new RolePrincipalAssociator(PolicyRefUpdater.PRINCIPAL_TYPE.ROLE, roleRole, roleId); if (!associator.doAssociate(false)) { - throw restErrorUtil.createRESTException("Role with name: " + roleRole + " does not exist ", MessageEnums.INVALID_INPUT_DATA); + if (isCreateNonExistentUGRs) { + rangerTransactionSynchronizationAdapter.executeOnTransactionCommit(associator); + } else { + throw restErrorUtil.createRESTException("Role with name: " + roleRole + " does not exist ", MessageEnums.INVALID_INPUT_DATA); + } } } } @@ -273,7 +280,7 @@ private Long createOrGetPrincipal(final boolean createIfAbsent) { } private Long createPrincipal(String user) { - LOG.warn("User specified in role does not exist in ranger admin, creating new user, Type: " + type.name() + ", name = " + user); + LOG.warn(type.name()+" specified in role does not exist in ranger admin, creating new "+type.name()+", Type: " + type.name() + ", name = " + user); if (LOG.isDebugEnabled()) { LOG.debug("===> RolePrincipalAssociator.createPrincipal(type=" + type.name() +", name=" + name + ")"); @@ -312,6 +319,17 @@ private Long createPrincipal(String user) { } } break; + case ROLE: { + // Create role + try { + RangerRole rRole = new RangerRole(name, null, null, null, null); + RangerRole createdRole = roleStore.createRole(rRole, false); + ret = createdRole.getId(); + } catch (Exception e) { + LOG.error("Failed to create Role "+ type.name()); + } + } + break; default: break; } diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java index 1f9801ba2d..ed1ea03768 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java @@ -182,6 +182,7 @@ import org.apache.ranger.service.XUserService; import org.apache.ranger.util.RestUtil; import org.apache.ranger.view.RangerExportPolicyList; +import org.apache.ranger.view.RangerExportRoleList; import org.apache.ranger.view.RangerPolicyList; import org.apache.ranger.view.RangerServiceDefList; import org.apache.ranger.view.RangerServiceList; @@ -233,6 +234,7 @@ public class ServiceDBStore extends AbstractServiceStore { private static final String USER_NAME = "Exported by"; private static final String RANGER_VERSION = "Ranger apache version"; private static final String TIMESTAMP = "Export time"; + private static final String EXPORT_COUNT = "Exported count"; private static final String SERVICE_CHECK_USER = "service.check.user"; private static final String AMBARI_SERVICE_CHECK_USER = "ambari.service.check.user"; @@ -2490,15 +2492,26 @@ public void getPoliciesInCSV(List policies, } } } - - public void getPoliciesInJson(List policies, - HttpServletResponse response) throws Exception { + + public enum JSON_FILE_NAME_TYPE { POLICY, ROLE } + public void getObjectInJson(List objList, + HttpServletResponse response, JSON_FILE_NAME_TYPE type) throws Exception { if (LOG.isDebugEnabled()) { - LOG.debug("==> ServiceDBStore.getPoliciesInJson()"); + LOG.debug("==> ServiceDBStore.getObjectInJson()"); } String timeStamp = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()); - String jsonFileName = "Ranger_Policies_" + timeStamp + ".json"; - writeJson(policies, jsonFileName, response); + String jsonFileName; + switch(type) { + case POLICY : + jsonFileName = "Ranger_Policies_" + timeStamp + ".json"; + break; + case ROLE : + jsonFileName = "Ranger_Roles_" + timeStamp + ".json"; + break; + default : + throw restErrorUtil.createRESTException("Invalid type "+type); + } + writeJson(objList, jsonFileName, response, type); } public PList getPaginatedPolicies(SearchFilter filter) throws Exception { @@ -4491,30 +4504,47 @@ private void writeCSVForPolicyItems(RangerPolicy policy, csvBuffer.append(COMMA_DELIMITER); csvBuffer.append(LINE_SEPARATOR); } - - public void putMetaDataInfo(RangerExportPolicyList rangerExportPolicyList){ + + public Map getMetaDataInfo() { Map metaDataInfo = new LinkedHashMap(); UserSessionBase usb = ContextUtil.getCurrentUserSession(); String userId = usb!=null ? usb.getLoginId() : null; - + metaDataInfo.put(HOSTNAME, LOCAL_HOSTNAME); metaDataInfo.put(USER_NAME, userId); metaDataInfo.put(TIMESTAMP, MiscUtil.getUTCDateForLocalDate(new Date())); metaDataInfo.put(RANGER_VERSION, RangerVersionInfo.getVersion()); - - rangerExportPolicyList.setMetaDataInfo(metaDataInfo); + + return metaDataInfo; } - - private void writeJson(List policies, String jsonFileName, - HttpServletResponse response) throws JSONException, IOException { + + private void writeJson(List objList, String jsonFileName, + HttpServletResponse response, JSON_FILE_NAME_TYPE type) throws JSONException, IOException { response.setContentType("text/json"); response.setHeader("Content-Disposition", "attachment; filename="+ jsonFileName); ServletOutputStream out = null; - RangerExportPolicyList rangerExportPolicyList = new RangerExportPolicyList(); - putMetaDataInfo(rangerExportPolicyList); - rangerExportPolicyList.setPolicies(policies); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); - String json = gson.toJson(rangerExportPolicyList, RangerExportPolicyList.class); + String json = null; + + switch(type) { + case POLICY : + RangerExportPolicyList rangerExportPolicyList = new RangerExportPolicyList(); + rangerExportPolicyList.setGenericPolicies(objList); + rangerExportPolicyList.setMetaDataInfo(getMetaDataInfo()); + json = gson.toJson(rangerExportPolicyList, RangerExportPolicyList.class); + break; + case ROLE : + RangerExportRoleList rangerExportRoleList = new RangerExportRoleList(); + rangerExportRoleList.setGenericRoleList(objList); + Map metaDataInfo = getMetaDataInfo(); + metaDataInfo.put(EXPORT_COUNT,rangerExportRoleList.getListSize()); + rangerExportRoleList.setMetaDataInfo(metaDataInfo); + json = gson.toJson(rangerExportRoleList, RangerExportRoleList.class); + break; + default : + throw restErrorUtil.createRESTException("Invalid type "+type); + } try { out = response.getOutputStream(); response.setStatus(HttpServletResponse.SC_OK); diff --git a/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java b/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java index 7fb7d9c790..a4dd6ef89a 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java @@ -19,6 +19,8 @@ package org.apache.ranger.rest; +import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -30,15 +32,18 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.*; import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; import org.apache.commons.lang.StringUtils; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; import org.apache.ranger.admin.client.datatype.RESTResponse; import org.apache.ranger.authorization.utils.StringUtil; import org.apache.ranger.biz.AssetMgr; import org.apache.ranger.biz.RangerBizUtil; import org.apache.ranger.biz.RoleDBStore; import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.biz.ServiceDBStore.JSON_FILE_NAME_TYPE; import org.apache.ranger.biz.XUserMgr; import org.apache.ranger.common.RESTErrorUtil; import org.apache.ranger.common.RangerSearchUtil; @@ -46,10 +51,12 @@ import org.apache.ranger.common.ServiceUtil; import org.apache.ranger.common.UserSessionBase; import org.apache.ranger.common.PropertiesUtil; +import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.ContextUtil; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.XXService; import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.entity.XXTrxLog; import org.apache.ranger.plugin.model.RangerPluginInfo; import org.apache.ranger.plugin.model.RangerRole; import org.apache.ranger.plugin.model.RangerService; @@ -58,20 +65,29 @@ import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; import org.apache.ranger.plugin.store.EmbeddedServiceDefsUtil; import org.apache.ranger.plugin.util.GrantRevokeRoleRequest; +import org.apache.ranger.plugin.util.JsonUtilsV2; import org.apache.ranger.plugin.util.RangerRESTUtils; import org.apache.ranger.plugin.util.RangerRoles; import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.security.context.RangerAdminOpContext; +import org.apache.ranger.security.context.RangerContextHolder; import org.apache.ranger.service.RangerRoleService; import org.apache.ranger.service.XUserService; +import org.apache.ranger.view.RangerExportRoleList; import org.apache.ranger.view.RangerRoleList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; +import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; +import com.google.gson.JsonSyntaxException; +import com.sun.jersey.multipart.FormDataParam; +import com.sun.jersey.core.header.FormDataContentDisposition; + @Path("roles") @Component @Scope("request") @@ -82,6 +98,8 @@ public class RoleREST { private static List INVALID_USERS = new ArrayList<>(); public static final String POLICY_DOWNLOAD_USERS = "policy.download.auth.users"; + public static final String PARAM_ROLE_NAME = "roleName"; + public static final String PARAM_IMPORT_IN_PROGRESS = "importInProgress"; @Autowired RESTErrorUtil restErrorUtil; @@ -363,6 +381,215 @@ public RangerRoleList getAllRoles(@Context HttpServletRequest request) { return ret; } + @GET + @Path("/roles/exportJson") + @Produces({ "application/json" }) + @PreAuthorize("@rangerPreAuthSecurityHandler.isAdminRole()") + public void getRolesInJson(@Context HttpServletRequest request, @Context HttpServletResponse response) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> getRolesInJson()"); + } + try { + List roleLists = getAllFilteredRoleList(request); + + if (CollectionUtils.isNotEmpty(roleLists)) { + svcStore.getObjectInJson(roleLists, response, JSON_FILE_NAME_TYPE.ROLE); + } else { + response.setStatus(HttpServletResponse.SC_NO_CONTENT); + LOG.error("There is no Role to Export!!"); + } + + } catch (WebApplicationException excp) { + throw excp; + } catch (Throwable excp) { + LOG.error("Error while exporting policy file!!", excp); + throw restErrorUtil.createRESTException(excp.getMessage()); + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== getRolesInJson()"); + } + } + + @POST + @Path("/roles/importRolesFromFile") + @Consumes({ MediaType.MULTIPART_FORM_DATA, MediaType.APPLICATION_JSON }) + @Produces({ "application/json", "application/xml" }) + @PreAuthorize("@rangerPreAuthSecurityHandler.isAdminRole()") + public RESTResponse importRolesFromFile(@Context HttpServletRequest request, + @FormDataParam("file") InputStream uploadedInputStream, + @FormDataParam("file") FormDataContentDisposition fileDetail, + @QueryParam("updateIfExists") Boolean updateIfExists, + @DefaultValue("false") @QueryParam("createNonExistUserGroupRole") Boolean createNonExistUserGroupRole) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RoleREST.importRolesFromFile()"); + } + RESTResponse ret = new RESTResponse(); + + RangerAdminOpContext opContext = new RangerAdminOpContext(); + opContext.setBulkModeContext(true); + RangerContextHolder.setOpContext(opContext); + String metaDataInfo = null; + List trxLogListError = new ArrayList(); + XXTrxLog xxTrxLogError = new XXTrxLog(); + request.setAttribute(PARAM_IMPORT_IN_PROGRESS, true); + + try { + List trxLogList = new ArrayList(); + XXTrxLog xxTrxLog = new XXTrxLog(); + xxTrxLog.setAction("IMPORT START"); + xxTrxLog.setObjectClassType(AppConstants.CLASS_TYPE_RANGER_ROLE); + xxTrxLog.setPreviousValue("IMPORT START"); + trxLogList.add(xxTrxLog); + bizUtil.createTrxLog(trxLogList); + + if (updateIfExists == null) { + updateIfExists = false; + } + List roleNameList = new ArrayList(); + + roleNameList = getRoleNameList(request, roleNameList); + + String fileName = fileDetail.getFileName(); + int totalRoleCreate = 0; + int totalRoleUpdate = 0; + int totalRoleUnchange = 0; + String msg; + + if (fileName.endsWith("json")) { + try { + RangerExportRoleList rangerExportRoleList = null; + List roles = null; + rangerExportRoleList = processRoleInputJsonForMetaData(uploadedInputStream, rangerExportRoleList); + + if (rangerExportRoleList != null + && !CollectionUtils.sizeIsEmpty(rangerExportRoleList.getMetaDataInfo())) { + metaDataInfo = JsonUtilsV2.mapToJson(rangerExportRoleList.getMetaDataInfo()); + } else { + LOG.info("metadata info is not provided!!"); + } + roles = getRolesFromProvidedJson(rangerExportRoleList); + + if (roles != null && !CollectionUtils.sizeIsEmpty(roles)) { + for (RangerRole roleInJson : roles) { + + if (roleInJson != null && StringUtils.isNotEmpty(roleInJson.getName().trim())) { + String roleNameInJson = roleInJson.getName().trim(); + if (CollectionUtils.isNotEmpty(roleNameList) && roleNameList.contains(roleNameInJson)) { + + // check updateIfExists + if (updateIfExists) { + try { + RangerRole exitingRole = roleStore.getRole(roleNameInJson); + if (!exitingRole.getId().equals(roleInJson.getId())) { + roleInJson.setId(exitingRole.getId()); + } + if(exitingRole.equals(roleInJson)){ + totalRoleUnchange++; + if (LOG.isDebugEnabled()) { + LOG.debug("Ignoring Roles from provided role in Json file... "+ roleNameInJson); + } + } + else { + roleStore.updateRole(roleInJson, createNonExistUserGroupRole); + totalRoleUpdate++; + } + } catch (WebApplicationException excp) { + throw excp; + } catch (Throwable excp) { + LOG.error("updateRole(" + roleInJson + ") failed", excp); + + throw restErrorUtil.createRESTException(excp.getMessage()); + } + } else { + totalRoleUnchange++; + if (LOG.isDebugEnabled()) { + LOG.debug("Ignoring Roles from provided role in Json file... " + roleNameInJson); + } + } + ret.setStatusCode(RESTResponse.STATUS_SUCCESS); + } else if (!roleNameList.contains(roleNameInJson) && (!roleNameInJson.isEmpty())) { + try { + roleStore.createRole(roleInJson, createNonExistUserGroupRole); + } catch (WebApplicationException excp) { + throw excp; + } catch (Throwable excp) { + LOG.error("createRole(" + roleInJson + ") failed", excp); + + throw restErrorUtil.createRESTException(excp.getMessage()); + } + totalRoleCreate++; + ret.setStatusCode(RESTResponse.STATUS_SUCCESS); + } + } + } + } else { + LOG.error("Json File does not contain any role."); + throw restErrorUtil.createRESTException("Json File does not contain any role."); + } + if (updateIfExists) { + msg = "Total Role Created = " + totalRoleCreate + " , Total Role Updated = " + totalRoleUpdate + " , Total Role Unchanged = " + totalRoleUnchange; + ret.setMsgDesc(msg); + } else { + msg = "Total Role Created = " + totalRoleCreate + " , Total Role Unchanged = " + totalRoleUnchange; + ret.setMsgDesc(msg); + } + + } catch (IOException e) { + LOG.error(e.getMessage()); + throw restErrorUtil.createRESTException(e.getMessage()); + } + } else { + LOG.error("Provided file format is not supported!!"); + throw restErrorUtil.createRESTException("Provided file format is not supported!!"); + } + } catch (JsonSyntaxException ex) { + LOG.error("Provided json file is not valid!!", ex); + xxTrxLogError.setAction("IMPORT ERROR"); + xxTrxLogError.setObjectClassType(AppConstants.CLASS_TYPE_RANGER_ROLE); + if (StringUtils.isNotEmpty(metaDataInfo)) { + xxTrxLogError.setPreviousValue(metaDataInfo); + } + trxLogListError.add(xxTrxLogError); + bizUtil.createTrxLog(trxLogListError); + throw restErrorUtil.createRESTException(ex.getMessage()); + } catch (WebApplicationException excp) { + LOG.error("Error while importing role from file!!", excp); + xxTrxLogError.setAction("IMPORT ERROR"); + xxTrxLogError.setObjectClassType(AppConstants.CLASS_TYPE_RANGER_ROLE); + if (StringUtils.isNotEmpty(metaDataInfo)) { + xxTrxLogError.setPreviousValue(metaDataInfo); + } + trxLogListError.add(xxTrxLogError); + bizUtil.createTrxLog(trxLogListError); + throw excp; + } catch (Throwable excp) { + LOG.error("Error while importing role from file!!", excp); + xxTrxLogError.setAction("IMPORT ERROR"); + xxTrxLogError.setObjectClassType(AppConstants.CLASS_TYPE_RANGER_ROLE); + if (StringUtils.isNotEmpty(metaDataInfo)) { + xxTrxLogError.setPreviousValue(metaDataInfo); + } + trxLogListError.add(xxTrxLogError); + bizUtil.createTrxLog(trxLogListError); + throw restErrorUtil.createRESTException(excp.getMessage()); + } finally { + List trxLogListEnd = new ArrayList(); + XXTrxLog xxTrxLogEnd = new XXTrxLog(); + xxTrxLogEnd.setAction("IMPORT END"); + xxTrxLogEnd.setObjectClassType(AppConstants.CLASS_TYPE_RANGER_ROLE); + if (StringUtils.isNotEmpty(metaDataInfo)) { + xxTrxLogEnd.setPreviousValue(metaDataInfo); + } + trxLogListEnd.add(xxTrxLogEnd); + bizUtil.createTrxLog(trxLogListEnd); + if (LOG.isDebugEnabled()) { + LOG.debug("<== RoleREST.importRolesFromFile()"); + } + } + + return ret; + } + @GET @Path("/lookup/roles") @Produces({ "application/json" }) @@ -1323,4 +1550,83 @@ private void validateUsersGroupsAndRoles(GrantRevokeRoleRequest request){ request.setRoles(new HashSet<>()); } } -} + + protected List getAllFilteredRoleList(HttpServletRequest request) throws Exception { + if (LOG.isDebugEnabled()) { + LOG.debug("==> getAllFilteredRoleList()"); + } + String roleNames = null; + List roleNameList = null; + List roleLists = new ArrayList<>(); + + if (request.getParameter(PARAM_ROLE_NAME) != null) { + roleNames = request.getParameter(PARAM_ROLE_NAME); + } + if (StringUtils.isNotEmpty(roleNames)) { + roleNameList = new ArrayList(Arrays.asList(roleNames.split(","))); + } + + List rangerRoleList = new ArrayList(); + SearchFilter filter = new SearchFilter(); + + rangerRoleList = roleStore.getRoles(filter); + + if (!CollectionUtils.isEmpty(rangerRoleList)) { + for (RangerRole role : rangerRoleList) { + if (role != null) { + if (CollectionUtils.isNotEmpty(roleNameList)) { + if (roleNameList.contains(role.getName())) { + // set createTime & updateTime Time as null since exported Roles don't need this + role.setCreateTime(null); + role.setUpdateTime(null); + roleLists.add(role); + roleNameList.remove(role.getName()); + if (roleNameList.size() == 0) { + break; + } + } + } else { + // set createTime & updateTime Time as null since exported Roles don't need this + role.setCreateTime(null); + role.setUpdateTime(null); + roleLists.add(role); + } + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== getAllFilteredRoleList()" + roleLists.size()); + } + return roleLists; + } + + private List getRoleNameList(HttpServletRequest request, List roleNameList) throws Exception { + SearchFilter filter = searchUtil.getSearchFilter(request, roleService.sortFields); + roleNameList = roleStore.getRoleNames(filter); + return roleNameList; + } + + private RangerExportRoleList processRoleInputJsonForMetaData(InputStream uploadedInputStream, + RangerExportRoleList rangerExportRoleList) throws Exception { + String rolesString = IOUtils.toString(uploadedInputStream); + rolesString = rolesString.trim(); + if (StringUtils.isNotEmpty(rolesString)) { + rangerExportRoleList = JsonUtilsV2.jsonToObj(rolesString, RangerExportRoleList.class); + } else { + LOG.error("Provided json file is empty!!"); + throw restErrorUtil.createRESTException("Provided json file is empty!!"); + } + return rangerExportRoleList; + } + + private List getRolesFromProvidedJson(RangerExportRoleList rangerExportRoleList) { + List roles = null; + if (rangerExportRoleList != null && !CollectionUtils.sizeIsEmpty(rangerExportRoleList.getSecurityRoles())) { + roles = rangerExportRoleList.getSecurityRoles(); + } else { + LOG.error("Provided json file does not contain any role!!"); + throw restErrorUtil.createRESTException("Provided json file does not contain any role!!"); + } + return roles; + } +} \ No newline at end of file diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index e91d3bc717..ce26b1d662 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -72,6 +72,7 @@ import org.apache.ranger.biz.RoleDBStore; import org.apache.ranger.biz.SecurityZoneDBStore; import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.biz.ServiceDBStore.JSON_FILE_NAME_TYPE; import org.apache.ranger.biz.ServiceMgr; import org.apache.ranger.biz.TagDBStore; import org.apache.ranger.biz.XUserMgr; @@ -2101,11 +2102,11 @@ public void getPoliciesInExcel(@Context HttpServletRequest request, response.setStatus(HttpServletResponse.SC_NO_CONTENT); LOG.error("No policies found to download!"); } - + RangerExportPolicyList rangerExportPolicyList = new RangerExportPolicyList(); - svcStore.putMetaDataInfo(rangerExportPolicyList); + rangerExportPolicyList.setMetaDataInfo(svcStore.getMetaDataInfo()); String metaDataInfo = JsonUtilsV2.mapToJson(rangerExportPolicyList.getMetaDataInfo()); - + List trxLogList = new ArrayList(); XXTrxLog xxTrxLog = new XXTrxLog(); xxTrxLog.setAction("EXPORT EXCEL"); @@ -2154,11 +2155,11 @@ public void getPoliciesInCsv(@Context HttpServletRequest request, @Context HttpS response.setStatus(HttpServletResponse.SC_NO_CONTENT); LOG.error("No policies found to download!"); } - + RangerExportPolicyList rangerExportPolicyList = new RangerExportPolicyList(); - svcStore.putMetaDataInfo(rangerExportPolicyList); + rangerExportPolicyList.setMetaDataInfo(svcStore.getMetaDataInfo()); String metaDataInfo = JsonUtilsV2.mapToJson(rangerExportPolicyList.getMetaDataInfo()); - + List trxLogList = new ArrayList(); XXTrxLog xxTrxLog = new XXTrxLog(); xxTrxLog.setAction("EXPORT CSV"); @@ -2208,18 +2209,18 @@ public void getPoliciesInJson(@Context HttpServletRequest request, } } bizUtil.blockAuditorRoleUser(); - svcStore.getPoliciesInJson(policyLists, response); + svcStore.getObjectInJson(policyLists, response, JSON_FILE_NAME_TYPE.POLICY); } else { checkPoliciesExists = true; response.setStatus(HttpServletResponse.SC_NO_CONTENT); LOG.error("There is no Policy to Export!!"); } - + if(!checkPoliciesExists){ RangerExportPolicyList rangerExportPolicyList = new RangerExportPolicyList(); - svcStore.putMetaDataInfo(rangerExportPolicyList); + rangerExportPolicyList.setMetaDataInfo(svcStore.getMetaDataInfo()); String metaDataInfo = JsonUtilsV2.mapToJson(rangerExportPolicyList.getMetaDataInfo()); - + List trxLogList = new ArrayList(); XXTrxLog xxTrxLog = new XXTrxLog(); xxTrxLog.setAction("EXPORT JSON"); diff --git a/security-admin/src/main/java/org/apache/ranger/security/context/RangerPreAuthSecurityHandler.java b/security-admin/src/main/java/org/apache/ranger/security/context/RangerPreAuthSecurityHandler.java index c823fba45e..80511c6918 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/context/RangerPreAuthSecurityHandler.java +++ b/security-admin/src/main/java/org/apache/ranger/security/context/RangerPreAuthSecurityHandler.java @@ -123,4 +123,15 @@ public boolean isAdminOrKeyAdminRole(){ gjResponse.setMsgDesc("User is not allowed to access the API"); throw restErrorUtil.generateRESTException(gjResponse); } + + public boolean isAdminRole(){ + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + if (userSession != null && userSession.isUserAdmin()) { + return true; + } + VXResponse gjResponse = new VXResponse(); + gjResponse.setStatusCode(HttpServletResponse.SC_FORBIDDEN); // assert user is authenticated. + gjResponse.setMsgDesc("User is not allowed to access the API"); + throw restErrorUtil.generateRESTException(gjResponse); + } } diff --git a/security-admin/src/main/java/org/apache/ranger/view/RangerExportRoleList.java b/security-admin/src/main/java/org/apache/ranger/view/RangerExportRoleList.java new file mode 100644 index 0000000000..9616c1e4a4 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/view/RangerExportRoleList.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.view; + +import java.util.LinkedHashMap; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerExportRoleList extends RangerRoleList implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + Map metaDataInfo = new LinkedHashMap(); + + public Map getMetaDataInfo() { + return metaDataInfo; + } + + public void setMetaDataInfo(Map metaDataInfo) { + this.metaDataInfo = metaDataInfo; + } + +} diff --git a/security-admin/src/main/java/org/apache/ranger/view/RangerPolicyList.java b/security-admin/src/main/java/org/apache/ranger/view/RangerPolicyList.java index 4799b3f03e..0fdc4580be 100644 --- a/security-admin/src/main/java/org/apache/ranger/view/RangerPolicyList.java +++ b/security-admin/src/main/java/org/apache/ranger/view/RangerPolicyList.java @@ -56,6 +56,10 @@ public void setPolicies(List policies) { this.policies = policies; } + public void setGenericPolicies(List policies) { + this.policies = (List) policies; + } + @Override public int getListSize() { if (policies != null) { diff --git a/security-admin/src/main/java/org/apache/ranger/view/RangerRoleList.java b/security-admin/src/main/java/org/apache/ranger/view/RangerRoleList.java index adbe93db67..f6d400f429 100644 --- a/security-admin/src/main/java/org/apache/ranger/view/RangerRoleList.java +++ b/security-admin/src/main/java/org/apache/ranger/view/RangerRoleList.java @@ -57,6 +57,10 @@ public void setRoleList(List roles) { this.roles = roles; } + public void setGenericRoleList(List roles) { + this.roles = (List) roles; + } + @Override public int getListSize() { if (roles != null) { diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestRoleREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestRoleREST.java index 217c1bba37..2da72a1ea5 100644 --- a/security-admin/src/test/java/org/apache/ranger/rest/TestRoleREST.java +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestRoleREST.java @@ -18,6 +18,7 @@ import org.apache.ranger.admin.client.datatype.RESTResponse; import org.apache.ranger.biz.*; +import org.apache.ranger.biz.ServiceDBStore.JSON_FILE_NAME_TYPE; import org.apache.ranger.common.*; import org.apache.ranger.db.*; import org.apache.ranger.entity.*; @@ -41,7 +42,13 @@ import org.mockito.junit.MockitoJUnitRunner; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import java.util.*; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import com.sun.jersey.core.header.FormDataContentDisposition; import static org.mockito.ArgumentMatchers.eq; @@ -51,6 +58,8 @@ public class TestRoleREST { private static final Long userId = 8L; private static final Long roleId = 9L; private static final String adminLoginID = "admin"; + private static final JSON_FILE_NAME_TYPE ROLE = JSON_FILE_NAME_TYPE.ROLE; + String importRoleTestFilePath = "./src/test/java/org/apache/ranger/rest/importRole/import_role_test_file.json"; @Mock RangerRole role; @@ -804,6 +813,387 @@ public void test17dGetSecureRangerRolesIfUpdated(){ } } + // empty request roles (requestParamRoles = 0, dbRoles = 5, return = all dbRoles) + @Test + public void test18GetRolesInJson() throws Exception { + // pre-requisites + List rangerRolesProcessed = new ArrayList<>(); + + rangerRolesProcessed.add(createRangerRole("role1", true)); + rangerRolesProcessed.add(createRangerRole("role2", false)); + rangerRolesProcessed.add(createRangerRole("role3", false)); + rangerRolesProcessed.add(createRangerRole("adm", true)); + rangerRolesProcessed.add(createRangerRole("user", false)); + + // mock + HttpServletRequest requestMock = Mockito.mock(HttpServletRequest.class); + HttpServletResponse responseMock = Mockito.mock(HttpServletResponse.class); + + // stubs + Mockito.when(roleRest.getAllFilteredRoleList(requestMock)).thenReturn(rangerRolesProcessed); + + // test + roleRest.getRolesInJson(requestMock, responseMock); + Mockito.verify(svcStore).getObjectInJson(rangerRolesProcessed, responseMock, ROLE); + } + + // non-empty request roles (requestParamRoles = 2, dbRoles = 5, return = 2 requestParamRoles) + @Test + public void test18bGetRolesInJson() throws Exception { + // pre-requisites + List rangerRolesProcessed = new ArrayList<>(); + + RangerRole admRole = createRangerRole("adm", true); + RangerRole userRole = createRangerRole("user", false); + + rangerRolesProcessed.add(admRole); + rangerRolesProcessed.add(userRole); + + // mock + HttpServletRequest requestMock = Mockito.mock(HttpServletRequest.class); + HttpServletResponse responseMock = Mockito.mock(HttpServletResponse.class); + + // stubs + Mockito.when(roleRest.getAllFilteredRoleList(requestMock)).thenReturn(rangerRolesProcessed); + + // test + roleRest.getRolesInJson(requestMock, responseMock); + Mockito.verify(svcStore).getObjectInJson(rangerRolesProcessed, responseMock, ROLE); + } + + // non-empty request roles (requestParamRoles = 3, dbRoles = 0, return = 0 dbRoles) + @Test + public void test18cGetRolesInJson() throws Exception { + // pre-requisites + List rangerRolesProcessed = new ArrayList<>(); + + // mock + HttpServletRequest requestMock = Mockito.mock(HttpServletRequest.class); + HttpServletResponse responseMock = Mockito.mock(HttpServletResponse.class); + + // stubs + Mockito.when(roleRest.getAllFilteredRoleList(requestMock)).thenReturn(rangerRolesProcessed); + + // test + roleRest.getRolesInJson(requestMock, responseMock); + Mockito.verify(svcStore, Mockito.never()).getObjectInJson(rangerRolesProcessed, responseMock, ROLE); + } + + // getAllFilteredRoleList throws Exception + @Test(expected = Throwable.class) + public void test18dGetRolesInJson() throws Exception { + // pre-requisites + List rangerRolesProcessed = new ArrayList<>(); + + // mock + HttpServletRequest requestMock = Mockito.mock(HttpServletRequest.class); + HttpServletResponse responseMock = Mockito.mock(HttpServletResponse.class); + + // stubs + Mockito.when(roleRest.getAllFilteredRoleList(requestMock)).thenThrow(new Throwable()); + + // test + Assert.assertThrows(Throwable.class, () -> roleRest.getRolesInJson(requestMock, responseMock)); + Mockito.verify(svcStore, Mockito.never()).getObjectInJson(rangerRolesProcessed, responseMock, ROLE); + Mockito.verify(restErrorUtil).createRESTException(Mockito.anyString()); + } + + // full match: requestParamRoles = 0, dbRoles = 5, return = all dbRoles + @Test + public void test19GetAllFilteredRoleList() throws Exception { + // pre-requisites + String requestParamRoles = ""; + List rangerRolesDb = new ArrayList<>(); + List rangerRolesProcessedExpected = new ArrayList<>(); + + rangerRolesDb.add(createRangerRole("role1", true)); + rangerRolesDb.add(createRangerRole("role2", false)); + rangerRolesDb.add(createRangerRole("role3", false)); + rangerRolesDb.add(createRangerRole("adm", true)); + rangerRolesDb.add(createRangerRole("user", false)); + + rangerRolesProcessedExpected.addAll(rangerRolesDb); + + // mock + HttpServletRequest requestMock = Mockito.mock(HttpServletRequest.class); + + // stubs + Mockito.when(requestMock.getParameter(RoleREST.PARAM_ROLE_NAME)).thenReturn(requestParamRoles); + Mockito.when(roleStore.getRoles(Mockito.any(SearchFilter.class))).thenReturn(rangerRolesDb); + + // test + List rangerRolesProcessedActual = roleRest.getAllFilteredRoleList(requestMock); + Assert.assertNotNull(rangerRolesProcessedActual); + Assert.assertEquals(rangerRolesProcessedActual.size(), rangerRolesProcessedExpected.size()); + Assert.assertEquals(rangerRolesProcessedActual, rangerRolesProcessedExpected); + } + + // partial match: requestParamRoles = 2, dbRoles = 5, match = 2 + @Test + public void test19bGetAllFilteredRoleList() throws Exception { + // pre-requisites + String requestParamRoles = "adm,user"; + List rangerRolesDb = new ArrayList<>(); + List rangerRolesProcessedExpected = new ArrayList<>(); + + rangerRolesDb.add(createRangerRole("role1", true)); + rangerRolesDb.add(createRangerRole("role2", false)); + rangerRolesDb.add(createRangerRole("role3", false)); + + RangerRole admRole = createRangerRole("adm", true); + RangerRole userRole = createRangerRole("user", false); + + rangerRolesDb.add(admRole); + rangerRolesDb.add(userRole); + + rangerRolesProcessedExpected.add(admRole); + rangerRolesProcessedExpected.add(userRole); + + // mock + HttpServletRequest requestMock = Mockito.mock(HttpServletRequest.class); + + // stubs + Mockito.when(requestMock.getParameter(RoleREST.PARAM_ROLE_NAME)).thenReturn(requestParamRoles); + Mockito.when(roleStore.getRoles(Mockito.any(SearchFilter.class))).thenReturn(rangerRolesDb); + + // test + List rangerRolesProcessedActual = roleRest.getAllFilteredRoleList(requestMock); + Assert.assertNotNull(rangerRolesProcessedActual); + Assert.assertEquals(rangerRolesProcessedActual.size(), rangerRolesProcessedExpected.size()); + Assert.assertEquals(rangerRolesProcessedActual, rangerRolesProcessedExpected); + } + + // partial match: requestParamRoles = 4, dbRoles = 5, match = 2 + @Test + public void test19cGetAllFilteredRoleList() throws Exception { + // pre-requisites + String requestParamRoles = "adm,key-adm,delegate-adm,user"; + List rangerRolesDb = new ArrayList<>(); + List rangerRolesProcessedExpected = new ArrayList<>(); + + rangerRolesDb.add(createRangerRole("role1", true)); + rangerRolesDb.add(createRangerRole("role2", false)); + rangerRolesDb.add(createRangerRole("role3", false)); + + RangerRole admRole = createRangerRole("adm", true); + RangerRole userRole = createRangerRole("user", false); + + rangerRolesDb.add(admRole); + rangerRolesDb.add(userRole); + + rangerRolesProcessedExpected.add(admRole); + rangerRolesProcessedExpected.add(userRole); + + // mock + HttpServletRequest requestMock = Mockito.mock(HttpServletRequest.class); + + // stubs + Mockito.when(requestMock.getParameter(RoleREST.PARAM_ROLE_NAME)).thenReturn(requestParamRoles); + Mockito.when(roleStore.getRoles(Mockito.any(SearchFilter.class))).thenReturn(rangerRolesDb); + + // test + List rangerRolesProcessedActual = roleRest.getAllFilteredRoleList(requestMock); + Assert.assertNotNull(rangerRolesProcessedActual); + Assert.assertEquals(rangerRolesProcessedActual.size(), rangerRolesProcessedExpected.size()); + Assert.assertEquals(rangerRolesProcessedActual, rangerRolesProcessedExpected); + } + + // no match: requestParamRoles = 3, dbRoles = 5, match = 0 + @Test + public void test19dGetAllFilteredRoleList() throws Exception { + // pre-requisites + String requestParamRoles = "sys-adm,key-adm,delegate-adm"; + List rangerRolesDb = new ArrayList<>(); + List rangerRolesProcessedExpected = new ArrayList<>(); + + rangerRolesDb.add(createRangerRole("role1", true)); + rangerRolesDb.add(createRangerRole("role2", false)); + rangerRolesDb.add(createRangerRole("role3", false)); + + RangerRole admRole = createRangerRole("adm", true); + RangerRole userRole = createRangerRole("user", false); + + rangerRolesDb.add(admRole); + rangerRolesDb.add(userRole); + + // mock + HttpServletRequest requestMock = Mockito.mock(HttpServletRequest.class); + + // stubs + Mockito.when(requestMock.getParameter(RoleREST.PARAM_ROLE_NAME)).thenReturn(requestParamRoles); + Mockito.when(roleStore.getRoles(Mockito.any(SearchFilter.class))).thenReturn(rangerRolesDb); + + // test + List rangerRolesProcessedActual = roleRest.getAllFilteredRoleList(requestMock); + Assert.assertNotNull(rangerRolesProcessedActual); + Assert.assertEquals(rangerRolesProcessedActual.size(), rangerRolesProcessedExpected.size()); + Assert.assertEquals(rangerRolesProcessedActual, rangerRolesProcessedExpected); + } + + // no match: requestParamRoles = 3, dbRoles = 0, match = 0 + @Test + public void test19eGetAllFilteredRoleList() throws Exception { + // pre-requisites + String requestParamRoles = "sys-adm,key-adm,delegate-adm"; + List rangerRolesDb = new ArrayList<>(); + List rangerRolesProcessedExpected = Collections.emptyList(); + + // mock + HttpServletRequest requestMock = Mockito.mock(HttpServletRequest.class); + + // stubs + Mockito.when(requestMock.getParameter(RoleREST.PARAM_ROLE_NAME)).thenReturn(requestParamRoles); + Mockito.when(roleStore.getRoles(Mockito.any(SearchFilter.class))).thenReturn(rangerRolesDb); + + // test + List rangerRolesProcessedActual = roleRest.getAllFilteredRoleList(requestMock); + Assert.assertNotNull(rangerRolesProcessedActual); + Assert.assertEquals(rangerRolesProcessedActual.size(), rangerRolesProcessedExpected.size()); + Assert.assertEquals(rangerRolesProcessedActual, rangerRolesProcessedExpected); + } + + // full match: requestParamRoles = null, dbRoles = 5, return = all dbRoles + @Test + public void test19fGetAllFilteredRoleList() throws Exception { + // pre-requisites + String requestParamRoles = null; + List rangerRolesDb = new ArrayList<>(); + List rangerRolesProcessedExpected = new ArrayList<>(); + + rangerRolesDb.add(createRangerRole("role1", true)); + rangerRolesDb.add(createRangerRole("role2", false)); + rangerRolesDb.add(createRangerRole("role3", false)); + rangerRolesDb.add(createRangerRole("adm", true)); + rangerRolesDb.add(createRangerRole("user", false)); + + rangerRolesProcessedExpected.addAll(rangerRolesDb); + + // mock + HttpServletRequest requestMock = Mockito.mock(HttpServletRequest.class); + + // stubs + Mockito.when(requestMock.getParameter(RoleREST.PARAM_ROLE_NAME)).thenReturn(requestParamRoles); + Mockito.when(roleStore.getRoles(Mockito.any(SearchFilter.class))).thenReturn(rangerRolesDb); + + // test + List rangerRolesProcessedActual = roleRest.getAllFilteredRoleList(requestMock); + Assert.assertNotNull(rangerRolesProcessedActual); + Assert.assertEquals(rangerRolesProcessedActual.size(), rangerRolesProcessedExpected.size()); + Assert.assertEquals(rangerRolesProcessedActual, rangerRolesProcessedExpected); + } + + // import role with updateIfExists=false and createNonExistUserGroupRole=false + @SuppressWarnings("unchecked") + @Test + public void test20importRolesFromFile() throws Exception { + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + + List roleList = createRoleList(); + RangerRole rangerRole = createRole(); + + SearchFilter filter = new SearchFilter(); + + File jsonRoleFile = new File(importRoleTestFilePath); + InputStream uploadedInputStream = new FileInputStream(jsonRoleFile); + FormDataContentDisposition fileDetail = FormDataContentDisposition.name("file").fileName(jsonRoleFile.getName()) + .size(uploadedInputStream.toString().length()).build(); + boolean updateIfExists = false; + boolean createNonExistUserGroupRole = false; + + Mockito.when(searchUtil.getSearchFilter(request, roleService.sortFields)).thenReturn(filter); + Mockito.when(roleStore.getRoleNames(Mockito.any(SearchFilter.class))).thenReturn(roleList); + Mockito.when(roleStore.createRole(Mockito.any(RangerRole.class), eq(createNonExistUserGroupRole))) + .thenReturn(rangerRole); + + RESTResponse resp = roleRest.importRolesFromFile(request, uploadedInputStream, fileDetail, updateIfExists, + createNonExistUserGroupRole); + Assert.assertNotNull(resp); + Assert.assertEquals(resp.getStatusCode(), RESTResponse.STATUS_SUCCESS); + Assert.assertEquals(resp.getMsgDesc(), "Total Role Created = 6 , Total Role Unchanged = 1"); + } + + // import role with updateIfExists=false and createNonExistUserGroupRole=true + @SuppressWarnings("unchecked") + @Test + public void test20bimportRolesFromFile() throws Exception { + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + + List roleList = createRoleList(); + RangerRole rangerRole = createRoleWithUsersAndGroups(); + + SearchFilter filter = new SearchFilter(); + + File jsonRoleFile = new File(importRoleTestFilePath); + InputStream uploadedInputStream = new FileInputStream(jsonRoleFile); + FormDataContentDisposition fileDetail = FormDataContentDisposition.name("file").fileName(jsonRoleFile.getName()) + .size(uploadedInputStream.toString().length()).build(); + boolean updateIfExists = false; + boolean createNonExistUserGroupRole = true; + + Mockito.when(searchUtil.getSearchFilter(request, roleService.sortFields)).thenReturn(filter); + Mockito.when(roleStore.getRoleNames(Mockito.any(SearchFilter.class))).thenReturn(roleList); + Mockito.when(roleStore.createRole(Mockito.any(RangerRole.class), eq(createNonExistUserGroupRole))) + .thenReturn(rangerRole); + + RESTResponse resp = roleRest.importRolesFromFile(request, uploadedInputStream, fileDetail, updateIfExists, + createNonExistUserGroupRole); + Assert.assertNotNull(resp); + Assert.assertEquals(resp.getStatusCode(), RESTResponse.STATUS_SUCCESS); + Assert.assertEquals(resp.getMsgDesc(), "Total Role Created = 6 , Total Role Unchanged = 1"); + } + + // import role with updateIfExists=true and createNonExistUserGroupRole=true + @SuppressWarnings("unchecked") + @Test + public void test20cimportRolesFromFileWithUpdate() throws Exception { + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + + List roleList = createRoleList(); + RangerRole rangerRole = createRole(); + + SearchFilter filter = new SearchFilter(); + + File jsonRoleFile = new File(importRoleTestFilePath); + InputStream uploadedInputStream = new FileInputStream(jsonRoleFile); + FormDataContentDisposition fileDetail = FormDataContentDisposition.name("file").fileName(jsonRoleFile.getName()) + .size(uploadedInputStream.toString().length()).build(); + boolean updateIfExists = true; + boolean createNonExistUserGroupRole = true; + + Mockito.when(searchUtil.getSearchFilter(request, roleService.sortFields)).thenReturn(filter); + Mockito.when(roleStore.getRoleNames(Mockito.any(SearchFilter.class))).thenReturn(roleList); + Mockito.when(roleStore.getRole(Mockito.anyString())).thenReturn(rangerRole); + Mockito.when(roleStore.createRole(Mockito.any(RangerRole.class), eq(createNonExistUserGroupRole))) + .thenReturn(rangerRole); + + RESTResponse resp = roleRest.importRolesFromFile(request, uploadedInputStream, fileDetail, updateIfExists, + createNonExistUserGroupRole); + Assert.assertNotNull(resp); + Assert.assertEquals(resp.getStatusCode(), RESTResponse.STATUS_SUCCESS); + Assert.assertEquals(resp.getMsgDesc(), + "Total Role Created = 6 , Total Role Updated = 1 , Total Role Unchanged = 0"); + } + + // import role throws exceptions + @SuppressWarnings("unchecked") + @Test(expected = Throwable.class) + public void test20dimportRolesFromFileWithUpdate() throws Exception { + HttpServletRequest request = Mockito.mock(HttpServletRequest.class); + + File jsonRoleFile = new File(importRoleTestFilePath); + InputStream uploadedInputStream = new FileInputStream(jsonRoleFile); + FormDataContentDisposition fileDetail = FormDataContentDisposition.name("file").fileName(jsonRoleFile.getName()) + .size(uploadedInputStream.toString().length()).build(); + boolean updateIfExists = false; + boolean createNonExistUserGroupRole = false; + + Mockito.when(roleStore.getRoleNames(Mockito.any(SearchFilter.class))).thenThrow(new Throwable()); + + Assert.assertThrows(Throwable.class, () -> roleRest.importRolesFromFile(request, uploadedInputStream, + fileDetail, updateIfExists, createNonExistUserGroupRole)); + Mockito.verify(restErrorUtil).createRESTException(Mockito.anyString()); + } + private RangerRole createRole(){ String name = "test-role"; String name2 = "admin"; @@ -930,4 +1320,13 @@ private GrantRevokeRoleRequest createGrantRevokeRoleRequest(){ roleRequest.setTargetRoles(new HashSet<>(Arrays.asList("role1","role2"))); return roleRequest; } + + private RangerRole createRangerRole(String name, boolean isAdmin) { + RangerRole.RoleMember roleMember = new RangerRole.RoleMember(name, isAdmin); + List usersList = Collections.singletonList(roleMember); + RangerRole rangerRole = new RangerRole(name, name, null, usersList, null); + rangerRole.setCreatedByUser(name); + rangerRole.setId(roleId); + return rangerRole; + } } diff --git a/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java b/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java index 9d17553a42..03ceb62802 100644 --- a/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java +++ b/security-admin/src/test/java/org/apache/ranger/rest/TestServiceREST.java @@ -41,6 +41,7 @@ import org.apache.ranger.biz.RangerBizUtil; import org.apache.ranger.biz.SecurityZoneDBStore; import org.apache.ranger.biz.ServiceDBStore; +import org.apache.ranger.biz.ServiceDBStore.JSON_FILE_NAME_TYPE; import org.apache.ranger.biz.ServiceMgr; import org.apache.ranger.biz.TagDBStore; import org.apache.ranger.biz.XUserMgr; @@ -1640,7 +1641,7 @@ public void test45exportPoliciesInJSON() throws Exception { Mockito.when(daoManager.getXXServiceDef().getById(xService.getType())).thenReturn(xServiceDef); serviceREST.getPoliciesInJson(request, response, false); - Mockito.verify(svcStore).getPoliciesInJson(rangerPolicyList, response); + Mockito.verify(svcStore).getObjectInJson(rangerPolicyList, response, JSON_FILE_NAME_TYPE.POLICY); } @Test diff --git a/security-admin/src/test/java/org/apache/ranger/rest/importRole/import_role_test_file.json b/security-admin/src/test/java/org/apache/ranger/rest/importRole/import_role_test_file.json new file mode 100644 index 0000000000..cf4e2b656e --- /dev/null +++ b/security-admin/src/test/java/org/apache/ranger/rest/importRole/import_role_test_file.json @@ -0,0 +1,120 @@ +{ + "metaDataInfo": { + "Host name": "dhaval-Vostro-15-3568", + "Exported by": "admin", + "Export time": "Jun 6, 2023 12:08:14 PM", + "Ranger apache version": "3.0.0-SNAPSHOT", + "Exported count": 7 + }, + "roles": [ + { + "name": "testrole1", + "description": "", + "options": {}, + "users": [ + { + "name": "testuser1", + "isAdmin": false + } + ], + "groups": [], + "roles": [], + "id": 1, + "isEnabled": true, + "createdBy": "Admin", + "updatedBy": "Admin" + }, + { + "name": "testrole2", + "description": "", + "options": {}, + "users": [], + "groups": [], + "roles": [], + "id": 2, + "isEnabled": true, + "createdBy": "Admin", + "updatedBy": "Admin" + }, + { + "name": "testrole3", + "description": "", + "options": {}, + "users": [ + { + "name": "testuser2", + "isAdmin": true + } + ], + "groups": [], + "roles": [], + "id": 3, + "isEnabled": true, + "createdBy": "Admin", + "updatedBy": "Admin" + }, + { + "name": "testrole4", + "description": "", + "options": {}, + "users": [ + { + "name": "testuser3", + "isAdmin": false + } + ], + "groups": [], + "roles": [], + "id": 4, + "isEnabled": true, + "createdBy": "Admin", + "updatedBy": "Admin" + }, + { + "name": "testrole5", + "description": "", + "options": {}, + "users": [], + "groups": [], + "roles": [], + "id": 5, + "isEnabled": true, + "createdBy": "Admin", + "updatedBy": "Admin" + }, + { + "name": "testrole6", + "description": "", + "options": {}, + "users": [], + "groups": [], + "roles": [], + "id": 6, + "isEnabled": true, + "createdBy": "Admin", + "updatedBy": "Admin" + }, + { + "name": "admin", + "description": "", + "options": {}, + "users": [ + { + "name": "testuser5", + "isAdmin": false + } + ], + "groups": [], + "roles": [], + "id": 7, + "isEnabled": true, + "createdBy": "Admin", + "updatedBy": "Admin" + } + ], + "startIndex": 0, + "pageSize": 0, + "totalCount": 0, + "resultSize": 0, + "queryTimeMS": 1686053294837 +} \ No newline at end of file From 433a2ffa44c12b278806daf61dca432a70636e74 Mon Sep 17 00:00:00 2001 From: Dineshkumar Yadav Date: Sat, 3 Jun 2023 18:40:30 +0530 Subject: [PATCH 03/46] RANGER-4151 : Create HA support for tagSync Signed-off-by: Dineshkumar Yadav --- distro/src/main/assembly/tagsync.xml | 7 ++ .../templates/ranger-tagsync-template.xml | 58 +++++++++ tagsync/pom.xml | 6 +- .../tagsync/ha/TagSyncHAInitializerImpl.java | 110 ++++++++++++++++++ .../tagsync/model/AbstractTagSource.java | 7 +- .../ranger/tagsync/process/TagSyncConfig.java | 17 ++- .../process/TagSyncMetricsProducer.java | 11 +- .../tagsync/process/TagSynchronizer.java | 9 ++ .../sink/tagadmin/TagAdminRESTSink.java | 51 ++++---- .../tagsync/source/atlas/AtlasTagSource.java | 10 +- .../source/atlasrest/AtlasRESTTagSource.java | 23 ++-- .../tagsync/source/file/FileTagSource.java | 7 +- .../main/resources/ranger-tagsync-site.xml | 58 ++++++++- 13 files changed, 329 insertions(+), 45 deletions(-) create mode 100644 tagsync/src/main/java/org/apache/ranger/tagsync/ha/TagSyncHAInitializerImpl.java diff --git a/distro/src/main/assembly/tagsync.xml b/distro/src/main/assembly/tagsync.xml index 44a8233ccc..a04026059c 100644 --- a/distro/src/main/assembly/tagsync.xml +++ b/distro/src/main/assembly/tagsync.xml @@ -27,6 +27,7 @@ true org.apache.ranger:ranger-tagsync + org.apache.ranger:ranger-common-ha dist @@ -95,6 +96,12 @@ org.slf4j:log4j-over-slf4j:jar:${slf4j.version} ch.qos.logback:logback-classic:jar:${logback.version} ch.qos.logback:logback-core:jar:${logback.version} + org.apache.ranger:ranger-common-ha:jar:${project.version} + org.apache.curator:curator-framework:jar:${curator.version} + org.apache.curator:curator-recipes:jar:${curator.version} + org.apache.curator:curator-client:jar:${curator.version} + org.apache.zookeeper:zookeeper:jar:${zookeeper.version} + org.apache.zookeeper:zookeeper-jute:jar:${zookeeper.version} diff --git a/tagsync/conf/templates/ranger-tagsync-template.xml b/tagsync/conf/templates/ranger-tagsync-template.xml index 40bd3dbe60..464dfc6a92 100644 --- a/tagsync/conf/templates/ranger-tagsync-template.xml +++ b/tagsync/conf/templates/ranger-tagsync-template.xml @@ -123,4 +123,62 @@ ranger.tagsync.metrics.enabled false + + + + ranger.service.name + ranger-tagsync + + + ranger-tagsync.server.ha.enabled + false + + + ranger-tagsync.server.ha.zookeeper.zkroot + /ranger-tagsync + + + ranger-tagsync.server.ha.zookeeper.connect + + + + ranger-tagsync.server.ha.ids + id1,id2 + + + ranger-tagsync.server.ha.address.id1 + + + + ranger-tagsync.server.ha.address.id2 + + + + ranger-tagsync.server.ha.zookeeper.retry.sleeptime.ms + 1000 + + + ranger-tagsync.server.ha.zookeeper.session.timeout.ms + 20000 + + + ranger-tagsync.server.ha.zookeeper.num.retries + 3 + + + ranger-tagsync.server.ha.zookeeper.acl + + + + ranger-tagsync.server.ha.zookeeper.auth + + + + ranger-tagsync.server.ha.http.port + + + + ranger-tagsync.server.ha.https.port + + diff --git a/tagsync/pom.xml b/tagsync/pom.xml index 1005a6d1ce..ede5e838ac 100644 --- a/tagsync/pom.xml +++ b/tagsync/pom.xml @@ -298,6 +298,10 @@ jackson-annotations ${atlas.jackson.version} - + + org.apache.ranger + ranger-common-ha + ${project.version} + diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/ha/TagSyncHAInitializerImpl.java b/tagsync/src/main/java/org/apache/ranger/tagsync/ha/TagSyncHAInitializerImpl.java new file mode 100644 index 0000000000..016d6e5ccd --- /dev/null +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/ha/TagSyncHAInitializerImpl.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.tagsync.ha; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.RangerHAInitializer; +import org.apache.ranger.ha.ActiveInstanceElectorService; +import org.apache.ranger.ha.ActiveStateChangeHandler; +import org.apache.ranger.ha.ServiceState; +import org.apache.ranger.ha.service.HARangerService; +import org.apache.ranger.ha.service.ServiceManager; +import org.apache.ranger.tagsync.process.TagSyncConfig; +import org.apache.log4j.Logger; + + +public class TagSyncHAInitializerImpl extends RangerHAInitializer { + private static final Logger LOG = Logger.getLogger(TagSyncHAInitializerImpl.class); + ActiveInstanceElectorService activeInstanceElectorService = null; + ActiveStateChangeHandler activeStateChangeHandler = null; + List haRangerService = null; + ServiceManager serviceManager = null; + private static TagSyncHAInitializerImpl theInstance = null; + + private TagSyncHAInitializerImpl(Configuration configuration) { + if(LOG.isDebugEnabled()){ + LOG.info("==> TagSyncHAInitializerImpl.TagSyncHAInitializerImpl()"); + } + try { + LOG.info("Ranger TagSync server is HA enabled : "+configuration.getBoolean(TagSyncConfig.TAGSYNC_SERVER_HA_ENABLED_PARAM, false) ); + init(configuration); + } catch (Exception e) { + LOG.error("TagSyncHAInitializerImpl initialization failed", e); + } + if(LOG.isDebugEnabled()){ + LOG.info("<== TagSyncHAInitializerImpl.TagSyncHAInitializerImpl()"); + } + } + + public void init(Configuration configuration) throws Exception { + super.init(configuration); + LOG.info("==> TagSyncHAInitializerImpl.init() initialization started"); + Set activeStateChangeHandlerProviders = new HashSet(); + activeInstanceElectorService = new ActiveInstanceElectorService(activeStateChangeHandlerProviders, + curatorFactory, activeInstanceState, serviceState, configuration); + + haRangerService = new ArrayList(); + haRangerService.add(activeInstanceElectorService); + serviceManager = new ServiceManager(haRangerService); + LOG.info("<== TagSyncHAInitializerImpl.init() initialization completed"); + } + + + @Override + public void stop() { + if(LOG.isDebugEnabled()){ + LOG.debug("==> TagSyncHAInitializerImpl.stop() "); + } + if (serviceManager != null) { + serviceManager.stop(); + } + if(curatorFactory != null){ + curatorFactory.close(); + } + if(LOG.isDebugEnabled()){ + LOG.debug("<== TagSyncHAInitializerImpl.stop() "); + } + } + + public static TagSyncHAInitializerImpl getInstance(Configuration configuration) { + if(theInstance == null){ + synchronized(TagSyncHAInitializerImpl.class){ + if(theInstance == null){ + theInstance = new TagSyncHAInitializerImpl(configuration); + } + } + } + return theInstance; + } + public boolean isActive() { + try { + // To let the curator thread a chance to run and set the active state if needed + Thread.sleep(0L); + } catch (InterruptedException exception) { + // Ignore + } + return serviceState.getState().equals(ServiceState.ServiceStateValue.ACTIVE); + } +} diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/model/AbstractTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/model/AbstractTagSource.java index 09f3292e75..ff9937628f 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/model/AbstractTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/model/AbstractTagSource.java @@ -21,6 +21,7 @@ import com.google.gson.Gson; import org.apache.ranger.plugin.util.ServiceTags; +import org.apache.ranger.tagsync.process.TagSyncConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,14 +60,16 @@ protected void updateSink(final ServiceTags toUpload) throws Exception { LOG.debug("No ServiceTags to upload"); } } else { + if(!TagSyncConfig.isTagSyncServiceActive()) { + LOG.error("This TagSync server is not in active state. Cannot commit transaction!"); + throw new RuntimeException("This TagSync server is not in active state. Cannot commit transaction!"); + } if (LOG.isDebugEnabled()) { String toUploadJSON = new Gson().toJson(toUpload); LOG.debug("Uploading serviceTags=" + toUploadJSON); } - try { ServiceTags uploaded = tagSink.upload(toUpload); - if (LOG.isDebugEnabled()) { String uploadedJSON = new Gson().toJson(uploaded); LOG.debug("Uploaded serviceTags=" + uploadedJSON); diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java index 590426cd0f..87b655df54 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java @@ -22,6 +22,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.SecureClientLogin; +import org.apache.ranger.tagsync.ha.TagSyncHAInitializerImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,6 +44,7 @@ public class TagSyncConfig extends Configuration { private static final Logger LOG = LoggerFactory.getLogger(TagSyncConfig.class); + private static TagSyncConfig instance = null; private static final String CONFIG_FILE = "ranger-tagsync-site.xml"; private static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default.xml"; @@ -122,7 +124,7 @@ public class TagSyncConfig extends Configuration { private static final String TAGSYNC_SINK_MAX_BATCH_SIZE_PROP = "ranger.tagsync.dest.ranger.max.batch.size"; private static final String TAGSYNC_ATLASREST_SOURCE_ENTITIES_BATCH_SIZE = "ranger.tagsync.source.atlasrest.entities.batch.size"; - + public static final String TAGSYNC_SERVER_HA_ENABLED_PARAM = "ranger-tagsync.server.ha.enabled"; private Properties props; @@ -135,7 +137,14 @@ public class TagSyncConfig extends Configuration { } public static TagSyncConfig getInstance() { - return new TagSyncConfig(); + if(instance == null ){ + synchronized (TagSyncConfig.class){ + if(instance == null ){ + instance = new TagSyncConfig(); + } + } + } + return instance; } public Properties getProperties() { @@ -215,6 +224,10 @@ public static String getResourceFileName(String path) { return ret; } + synchronized static public boolean isTagSyncServiceActive() { + return TagSyncHAInitializerImpl.getInstance(TagSyncConfig.getInstance()).isActive(); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncMetricsProducer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncMetricsProducer.java index a3b5d03e14..4b8decec34 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncMetricsProducer.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncMetricsProducer.java @@ -55,7 +55,7 @@ public void run() { + "] milliseconds before attempting to tagsync metrics information", e); } try { - writeJVMMetrics(logFileNameWithPath); + writeJVMMetrics(logFileNameWithPath, config); } catch (Throwable t) { LOG.error("Failed to write tagsync metrics into file. Error details: ", t); } @@ -71,7 +71,7 @@ public void run() { } - private void writeJVMMetrics(String logFileNameWithPath) throws Throwable { + private void writeJVMMetrics(String logFileNameWithPath, TagSyncConfig config) throws Throwable { try { File userMetricFile = null; userMetricFile = new File(logFileNameWithPath); @@ -79,6 +79,13 @@ private void writeJVMMetrics(String logFileNameWithPath) throws Throwable { userMetricFile.createNewFile(); } RangerMetricsUtil rangerMetricsUtil = new RangerMetricsUtil(); + if (config.getBoolean(TagSyncConfig.TAGSYNC_SERVER_HA_ENABLED_PARAM, false)) { + if(config.isTagSyncServiceActive()){ + rangerMetricsUtil.setIsRoleActive(1); + }else{ + rangerMetricsUtil.setIsRoleActive(0); + } + } rangerMetricsUtil.writeMetricsToFile(userMetricFile); } catch (Throwable t) { diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java index 55cc0a8aa2..7963f88c53 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java @@ -31,6 +31,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.security.SecureClientLogin; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.ranger.tagsync.ha.TagSyncHAInitializerImpl; import org.apache.ranger.tagsync.model.TagSink; import org.apache.ranger.tagsync.model.TagSource; import org.slf4j.Logger; @@ -52,6 +53,7 @@ public class TagSynchronizer { private final Object shutdownNotifier = new Object(); private volatile boolean isShutdownInProgress = false; + private TagSyncHAInitializerImpl tagSyncHAinitializerImpl = null; public static void main(String[] args) { TagSynchronizer tagSynchronizer = new TagSynchronizer(); @@ -63,16 +65,23 @@ public static void main(String[] args) { tagSynchronizer.setProperties(props); boolean tagSynchronizerInitialized = tagSynchronizer.initialize(); + tagSynchronizer.tagSyncHAinitializerImpl = TagSyncHAInitializerImpl.getInstance(config); if (tagSynchronizerInitialized) { try { tagSynchronizer.run(); } catch (Throwable t) { LOG.error("main thread caught exception..:", t); + if (tagSynchronizer.tagSyncHAinitializerImpl != null) { + tagSynchronizer.tagSyncHAinitializerImpl.stop(); + } System.exit(1); } } else { LOG.error("TagSynchronizer failed to initialize correctly, exiting.."); + if (tagSynchronizer.tagSyncHAinitializerImpl != null) { + tagSynchronizer.tagSyncHAinitializerImpl.stop(); + } System.exit(1); } diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java b/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java index db76a678f3..ac0069a939 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java @@ -366,40 +366,41 @@ public void run() { } while (true) { - UploadWorkItem uploadWorkItem; + if (TagSyncConfig.isTagSyncServiceActive()) { + UploadWorkItem uploadWorkItem; - try { - uploadWorkItem = uploadWorkItems.take(); + try { + uploadWorkItem = uploadWorkItems.take(); - ServiceTags toUpload = uploadWorkItem.getServiceTags(); + ServiceTags toUpload = uploadWorkItem.getServiceTags(); - boolean doRetry; + boolean doRetry; - do { - doRetry = false; + do { + doRetry = false; - try { - ServiceTags uploaded = doUpload(toUpload); - if (uploaded == null) { // Treat this as if an Exception is thrown by doUpload + try { + ServiceTags uploaded = doUpload(toUpload); + if (uploaded == null) { // Treat this as if an Exception is thrown by doUpload + doRetry = true; + Thread.sleep(rangerAdminConnectionCheckInterval); + } else { + // ServiceTags uploaded successfully + uploadWorkItem.uploadCompleted(uploaded); + } + } catch (InterruptedException interrupted) { + LOG.error("Caught exception..: ", interrupted); + return; + } catch (Exception exception) { doRetry = true; Thread.sleep(rangerAdminConnectionCheckInterval); - } else { - // ServiceTags uploaded successfully - uploadWorkItem.uploadCompleted(uploaded); } - } catch (InterruptedException interrupted) { - LOG.error("Caught exception..: ", interrupted); - return; - } catch (Exception exception) { - doRetry = true; - Thread.sleep(rangerAdminConnectionCheckInterval); - } - } while (doRetry); + } while (doRetry); - } - catch (InterruptedException exception) { - LOG.error("Interrupted..: ", exception); - return; + } catch (InterruptedException exception) { + LOG.error("Interrupted..: ", exception); + return; + } } } diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java index 34a39f73c9..071f52c4aa 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java @@ -184,9 +184,12 @@ public void run() { } while (true) { - - try { - List> newMessages = consumer.receive(MAX_WAIT_TIME_IN_MILLIS); + if (TagSyncConfig.isTagSyncServiceActive()) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> ConsumerRunnable.run() is running as server is active"); + } + try { + List> newMessages = consumer.receive(MAX_WAIT_TIME_IN_MILLIS); if (newMessages.size() == 0) { if (LOG.isDebugEnabled()) { @@ -251,6 +254,7 @@ public void run() { return; } } + } } } diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java index 9063b03f5b..792ced1329 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java @@ -189,15 +189,23 @@ public void run() { if (LOG.isDebugEnabled()) { LOG.debug("==> AtlasRESTTagSource.run()"); } - while (true) { - try { - synchUp(); - - LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); + while (true) { + try { + if (TagSyncConfig.isTagSyncServiceActive()) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> AtlasRESTTagSource.run() is running as server is Active"); + } + synchUp(); + }else{ + if (LOG.isDebugEnabled()) { + LOG.debug("==> This server is running passive mode"); + } + } + LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds"); - Thread.sleep(sleepTimeBetweenCycleInMillis); + Thread.sleep(sleepTimeBetweenCycleInMillis); - } catch (InterruptedException exception) { + } catch (InterruptedException exception) { LOG.error("Interrupted..: ", exception); return; } catch (Exception e) { @@ -208,7 +216,6 @@ public void run() { } public void synchUp() throws Exception { - List rangerAtlasEntities = getAtlasActiveEntities(); if (CollectionUtils.isNotEmpty(rangerAtlasEntities)) { diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java index 8af15f95a6..65deccd14a 100644 --- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java +++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/file/FileTagSource.java @@ -216,7 +216,12 @@ public void run() { while (true) { try { - synchUp(); + if (TagSyncConfig.isTagSyncServiceActive()) { + if (LOG.isDebugEnabled()) { + LOG.debug("==> FileTagSource is running as server is active"); + } + synchUp(); + } } catch (Exception e) { LOG.error("Caught exception..", e); } finally { diff --git a/tagsync/src/main/resources/ranger-tagsync-site.xml b/tagsync/src/main/resources/ranger-tagsync-site.xml index e17bdb213e..6bda5f4ab7 100644 --- a/tagsync/src/main/resources/ranger-tagsync-site.xml +++ b/tagsync/src/main/resources/ranger-tagsync-site.xml @@ -135,5 +135,61 @@ cl2_hdfs --> - + + + ranger.service.name + ranger-tagsync + + + ranger-tagsync.server.ha.enabled + false + + + ranger-tagsync.server.ha.zookeeper.zkroot + /ranger-tagsync + + + ranger-tagsync.server.ha.zookeeper.connect + + + + ranger-tagsync.server.ha.ids + id1,id2 + + + ranger-tagsync.server.ha.address.id1 + + + + ranger-tagsync.server.ha.address.id2 + + + + ranger-tagsync.server.ha.zookeeper.retry.sleeptime.ms + 1000 + + + ranger-tagsync.server.ha.zookeeper.session.timeout.ms + 20000 + + + ranger-tagsync.server.ha.zookeeper.num.retries + 3 + + + ranger-tagsync.server.ha.zookeeper.acl + + + + ranger-tagsync.server.ha.zookeeper.auth + + + + ranger-tagsync.server.ha.http.port + + + + ranger-tagsync.server.ha.https.port + + From 9a132c5d017c4f0a08837180625786a54539b758 Mon Sep 17 00:00:00 2001 From: Brijesh Bhalala Date: Wed, 12 Jul 2023 14:57:44 +0530 Subject: [PATCH 04/46] RANGER-4132: If view policy button is clicked for a policy which is deleted, then the page gets stuck in loading state Signed-off-by: Dineshkumar Yadav --- .../java/org/apache/ranger/biz/XUserMgr.java | 8 +++++ .../apache/ranger/common/RESTErrorUtil.java | 31 +++++++++++++++++++ .../java/org/apache/ranger/rest/RoleREST.java | 6 +++- .../apache/ranger/rest/SecurityZoneREST.java | 5 ++- .../org/apache/ranger/rest/ServiceREST.java | 5 ++- .../webapp/react-webapp/src/utils/fetchAPI.js | 2 -- .../views/AuditEvent/AdminLogs/PolicyLogs.jsx | 4 ++- .../AdminLogs/PolicyViewDetails.jsx | 10 +++--- .../PolicyListing/AddUpdatePolicyForm.jsx | 2 +- .../users_details/EditUserView.jsx | 2 +- 10 files changed, 62 insertions(+), 13 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java index f653dbc8a4..09154491dd 100755 --- a/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/XUserMgr.java @@ -2031,6 +2031,10 @@ public void deleteXGroup(Long id, boolean force) { xaBizUtil.blockAuditorRoleUser(); XXGroupDao xXGroupDao = daoManager.getXXGroup(); XXGroup xXGroup = xXGroupDao.getById(id); + if (xXGroup == null) { + throw restErrorUtil.create404RESTException("Data Not Found for given Id", MessageEnums.DATA_NOT_FOUND, id, + null, "readResource : No Object found with given id."); + } VXGroup vXGroup = xGroupService.populateViewBean(xXGroup); if (vXGroup == null || StringUtils.isEmpty(vXGroup.getName())) { throw restErrorUtil.createRESTException("Group ID doesn't exist.", MessageEnums.INVALID_INPUT_DATA); @@ -2249,6 +2253,10 @@ public synchronized void deleteXUser(Long id, boolean force) { xaBizUtil.blockAuditorRoleUser(); XXUserDao xXUserDao = daoManager.getXXUser(); XXUser xXUser = xXUserDao.getById(id); + if (xXUser == null) { + throw restErrorUtil.create404RESTException("Data Not Found for given Id", MessageEnums.DATA_NOT_FOUND, id, + null, "readResource : No Object found with given id."); + } VXUser vXUser = xUserService.populateViewBean(xXUser); if(vXUser==null || StringUtils.isEmpty(vXUser.getName())){ throw restErrorUtil.createRESTException("No user found with id=" + id); diff --git a/security-admin/src/main/java/org/apache/ranger/common/RESTErrorUtil.java b/security-admin/src/main/java/org/apache/ranger/common/RESTErrorUtil.java index e0ff958144..4aaf364428 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/RESTErrorUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RESTErrorUtil.java @@ -413,4 +413,35 @@ public WebApplicationException createRESTException(String errorMessage, restException); return restException; } + + public WebApplicationException create404RESTException(String errorMessage, + MessageEnums messageEnum, Long objectId, String fieldName, + String logMessage) { + List messageList = new ArrayList(); + messageList.add(messageEnum.getMessage(objectId, fieldName)); + + VXResponse gjResponse = new VXResponse(); + gjResponse.setStatusCode(VXResponse.STATUS_ERROR); + gjResponse.setMsgDesc(errorMessage); + gjResponse.setMessageList(messageList); + + Response errorResponse = Response + .status(javax.servlet.http.HttpServletResponse.SC_NOT_FOUND) + .entity(gjResponse).build(); + + WebApplicationException restException = new WebApplicationException( + errorResponse); + restException.fillInStackTrace(); + UserSessionBase userSession = ContextUtil.getCurrentUserSession(); + String loginId = null; + if (userSession != null) { + loginId = userSession.getLoginId(); + } + + logger.info("Request failed. loginId=" + + loginId + ", logMessage=" + gjResponse.getMsgDesc(), + restException); + + return restException; + } } diff --git a/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java b/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java index a4dd6ef89a..ca9f286b3d 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java @@ -53,6 +53,7 @@ import org.apache.ranger.common.PropertiesUtil; import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.ContextUtil; +import org.apache.ranger.common.MessageEnums; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.XXService; import org.apache.ranger.entity.XXServiceDef; @@ -287,7 +288,10 @@ public void deleteRole(@PathParam("id") Long roleId) { } catch(Throwable excp) { LOG.error("deleteRole(" + roleId + ") failed", excp); - throw restErrorUtil.createRESTException(excp.getMessage()); + throw restErrorUtil.createRESTException( + "Data Not Found for given Id", + MessageEnums.DATA_NOT_FOUND, roleId, null, + "readResource : No Object found with given id."); } if (LOG.isDebugEnabled()) { LOG.debug("<== deleteRole(id=" + roleId + ")"); diff --git a/security-admin/src/main/java/org/apache/ranger/rest/SecurityZoneREST.java b/security-admin/src/main/java/org/apache/ranger/rest/SecurityZoneREST.java index 66fa221638..55d6aaac50 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/SecurityZoneREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/SecurityZoneREST.java @@ -241,7 +241,10 @@ public void deleteSecurityZone(@PathParam("id") Long zoneId) { } catch(Throwable excp) { LOG.error("deleteSecurityZone(" + zoneId + ") failed", excp); - throw restErrorUtil.createRESTException(excp.getMessage()); + throw restErrorUtil.createRESTException( + "Data Not Found for given Id", + MessageEnums.DATA_NOT_FOUND, zoneId, null, + "readResource : No Object found with given id."); } if (LOG.isDebugEnabled()) { LOG.debug("<== deleteSecurityZone(id=" + zoneId + ")"); diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index ce26b1d662..a307293eb4 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -4569,7 +4569,10 @@ private String deleteServiceById(Long id) { svcStore.deleteService(id); } else { LOG.error("Cannot retrieve service:[" + id + "] for deletion"); - throw new Exception("deleteService(" + id + ") failed"); + throw restErrorUtil.createRESTException( + "Data Not Found for given Id", + MessageEnums.DATA_NOT_FOUND, id, null, + "readResource : No Object found with given id."); } } else { LOG.error("Cannot retrieve user session."); diff --git a/security-admin/src/main/webapp/react-webapp/src/utils/fetchAPI.js b/security-admin/src/main/webapp/react-webapp/src/utils/fetchAPI.js index 5d9a7d5c4e..ef32634bb4 100644 --- a/security-admin/src/main/webapp/react-webapp/src/utils/fetchAPI.js +++ b/security-admin/src/main/webapp/react-webapp/src/utils/fetchAPI.js @@ -97,11 +97,9 @@ async function fetchApi(axiosConfig = {}, otherConf = {}) { } if (error?.response?.status === 404) { navigateTo.navigate("/pageNotFound", { replace: true }); - return; } if (error?.response?.status === 403) { navigateTo.navigate("/forbidden", { replace: true }); - return; } throw error; } diff --git a/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AdminLogs/PolicyLogs.jsx b/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AdminLogs/PolicyLogs.jsx index abb58c94da..9c81e307da 100644 --- a/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AdminLogs/PolicyLogs.jsx +++ b/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AdminLogs/PolicyLogs.jsx @@ -1867,7 +1867,9 @@ export const PolicyLogs = ({ data, reportdata }) => {
Policy Name: {objectName}
-
Service Name: {owner}
+
+ Service Name: {parentObjectName} +
Created Date: {dateFormat(createDate, "mm/dd/yyyy hh:MM:ss TT ")} India Standard Time diff --git a/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AdminLogs/PolicyViewDetails.jsx b/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AdminLogs/PolicyViewDetails.jsx index d74f9a83e8..1f89900eb2 100644 --- a/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AdminLogs/PolicyViewDetails.jsx +++ b/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AdminLogs/PolicyViewDetails.jsx @@ -76,10 +76,10 @@ export function PolicyViewDetails(props) { } catch (error) { console.error(`eventTime can not be undefined ${error}`); } - accessLogsServiceDef = allServiceDefs.find((servicedef) => { - return servicedef.name == accesslogs.data.serviceType; + accessLogsServiceDef = allServiceDefs?.find((servicedef) => { + return servicedef.name == accesslogs?.data?.serviceType; }); - setAccess(accesslogs.data); + setAccess(accesslogs?.data); setServiceDef(accessLogsServiceDef); SetLoader(false); }; @@ -95,7 +95,7 @@ export function PolicyViewDetails(props) { } catch (error) { console.error(`versionNo can not be undefined ${error}`); } - setAccess(accesslogs.data); + setAccess(accesslogs?.data); }; const fetchPolicyVersions = async () => { @@ -115,7 +115,7 @@ export function PolicyViewDetails(props) { ); serverError(error); } - setAccess(accesslogs && accesslogs.data); + setAccess(accesslogs?.data); }; const { diff --git a/security-admin/src/main/webapp/react-webapp/src/views/PolicyListing/AddUpdatePolicyForm.jsx b/security-admin/src/main/webapp/react-webapp/src/views/PolicyListing/AddUpdatePolicyForm.jsx index cc34488ea0..13a8eaf4f9 100644 --- a/security-admin/src/main/webapp/react-webapp/src/views/PolicyListing/AddUpdatePolicyForm.jsx +++ b/security-admin/src/main/webapp/react-webapp/src/views/PolicyListing/AddUpdatePolicyForm.jsx @@ -256,7 +256,7 @@ export default function AddUpdatePolicyForm(props) { }); data = resp.data || null; } catch (error) { - console.error(`Error occurred while fetching service details ! ${error}`); + console.error(`Error occurred while fetching policy details ! ${error}`); } return data; }; diff --git a/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/users_details/EditUserView.jsx b/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/users_details/EditUserView.jsx index e47fccc6ff..73585458a9 100644 --- a/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/users_details/EditUserView.jsx +++ b/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/users_details/EditUserView.jsx @@ -79,7 +79,7 @@ function AddUserView(props) { }); } catch (error) { console.error( - `Error occurred while fetching Zones or CSRF headers! ${error}` + `Error occurred while fetching Users or CSRF headers! ${error}` ); } dispatch({ From 73ce97e0c93177cdc0be8ea90857765014e40047 Mon Sep 17 00:00:00 2001 From: Madhan Neethiraj Date: Thu, 13 Jul 2023 12:43:11 -0700 Subject: [PATCH 05/46] RANGER-4316: path recursive matcher fix to correctly handle path ending with separator --- .../RangerPathResourceMatcher.java | 9 ++-- .../RangerPathResourceMatcherTest.java | 44 +++++++++++-------- .../test_resourcematcher_path.json | 4 +- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java index 1af967fbd7..3c1523c254 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcher.java @@ -24,7 +24,6 @@ import org.apache.commons.io.IOCase; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; -import org.apache.ranger.plugin.policyengine.RangerAccessRequest; import org.apache.ranger.plugin.util.ServiceDefUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -189,8 +188,12 @@ static boolean isRecursiveWildCardMatch(String pathToCheck, String wildcardPath, sb.append(pathSeparatorChar); pathElementIndex++; } - if (useStringMatching && pathElements.length == wildcardPathElements.length) { // Loop finished normally and all sub-paths string-matched.. - ret = true; + if (useStringMatching) { + if (pathElements.length == wildcardPathElements.length) { // Loop finished normally and all sub-paths string-matched.. + ret = true; + } else if (pathToCheck.charAt(pathToCheck.length() - 1) == pathSeparatorChar) { // pathToCheck ends with separator, like /home/ + ret = pathElements.length == (wildcardPathElements.length - 1) && WILDCARD_ASTERISK.equals(wildcardPathElements[wildcardPathElements.length - 1]); + } } sb = null; diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcherTest.java b/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcherTest.java index 5e8efb7208..ed02be6748 100644 --- a/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcherTest.java +++ b/agents-common/src/test/java/org/apache/ranger/plugin/resourcematcher/RangerPathResourceMatcherTest.java @@ -35,24 +35,32 @@ public class RangerPathResourceMatcherTest { Object[][] data = { - // { resource, policy, optWildcard, recursive, result - { "/app/hive/test.db", "/", true, false, false, "user" }, - { "/app/hive/test.db", "/", true, true, true, "user" }, - { "/app/hive/test.db", "/*", true, false, true, "user" }, - { "/app/hbase/test.tbl", "/*", true, false, true, "user" }, - { "/app/hive/test.db", "/app", true, false, false, "user" }, - { "/app/hive/test.db", "/app/", true, false, false, "user" }, - { "/app/hive/test.db", "/app/", true, true, true, "user" }, - { "/app/hive/test.db", "/app/*", true, false, true, "user" }, - { "/app/hbase/test.tbl", "/app/*", true, false, true, "user" }, - { "/app/hive/test.db", "/app/hive/*", true, false, true, "user" }, - { "/app/hbase/test.tbl", "/app/hive/*", true, false, false, "user" }, - { "/app/hive/test.db", "/app/hive/test*", true, false, true, "user" }, - { "/app/hbase/test.tbl", "/app/hive/test*", true, false, false, "user" }, - { "/app/hive/test.db", "/app/hive/test.db", true, false, true, "user" }, - { "/app/hbase/test.tbl", "/app/hive/test.db", true, false, false, "user" }, - { "app/hive/*", "app/hive/*", false, false, true, "user" }, // simple string match - { "app/hive/test.db", "app/hive/*", false, false, false, "user" }, // simple string match + // resource policy wildcard recursive result user + { "/app/hive/test.db", "/", true, false, false, "user" }, + { "/app/hive/test.db", "/", true, true, true, "user" }, + { "/app/hive/test.db", "/*", true, false, true, "user" }, + { "/app/hbase/test.tbl", "/*", true, false, true, "user" }, + { "/app/hive/test.db", "/app", true, false, false, "user" }, + { "/app/hive/test.db", "/app/", true, false, false, "user" }, + { "/app/hive/test.db", "/app/", true, true, true, "user" }, + { "/app/hive/test.db", "/app/*", true, false, true, "user" }, + { "/app/hbase/test.tbl", "/app/*", true, false, true, "user" }, + { "/app/hive/test.db", "/app/hive/*", true, false, true, "user" }, + { "/app/hbase/test.tbl", "/app/hive/*", true, false, false, "user" }, + { "/app/hive/test.db", "/app/hive/test*", true, false, true, "user" }, + { "/app/hbase/test.tbl", "/app/hive/test*", true, false, false, "user" }, + { "/app/hive/test.db", "/app/hive/test.db", true, false, true, "user" }, + { "/app/hbase/test.tbl", "/app/hive/test.db", true, false, false, "user" }, + { "app/hive/*", "app/hive/*", false, false, true, "user" }, // simple string match + { "app/hive/test.db", "app/hive/*", false, false, false, "user" }, // simple string match + { "/app/", "/app/", true, true, true, "user" }, + { "/app/", "/app/", true, false, true, "user" }, + { "/app", "/app/", true, true, false, "user" }, + { "/app", "/app/", true, false, false, "user" }, + { "/app/", "/app/*", true, true, true, "user" }, + { "/app/", "/app/*", true, false, true, "user" }, + { "/app", "/app/*", true, true, false, "user" }, + { "/app", "/app/*", true, false, false, "user" }, }; Object[][] dataForSelfOrChildScope = { diff --git a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_path.json b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_path.json index 97765f94d1..7af8c92d50 100644 --- a/agents-common/src/test/resources/resourcematcher/test_resourcematcher_path.json +++ b/agents-common/src/test/resources/resourcematcher/test_resourcematcher_path.json @@ -180,7 +180,7 @@ "isRecursive": true }, "tests": [ - {"name": "seemingly-correct-path", "input": "/home/", "result": false}, + {"name": "seemingly-correct-path", "input": "/home/", "result": true}, {"name": "correct-path", "input": "/home/a.txt", "result": true} ] }, @@ -265,7 +265,7 @@ }, "policyResource": {"values": ["/home/"], "isRecursive": true}, "tests": [ - {"name": "slash-at-end-path", "input": "/home/", "result": false}, + {"name": "slash-at-end-path", "input": "/home/", "result": true}, {"name": "correct-path", "input": "/home/a.txt", "result": true}, {"name": "incomplete-path", "input": "/home", "result": false} ] From 807b514509c3d9a37ca86175057e6748bfd7fc79 Mon Sep 17 00:00:00 2001 From: szymonorz Date: Wed, 19 Jul 2023 18:52:06 +0200 Subject: [PATCH 06/46] RANGER-4322: fix enable-atlas-plugin.sh failure Signed-off-by: Madhan Neethiraj --- distro/src/main/assembly/plugin-atlas.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distro/src/main/assembly/plugin-atlas.xml b/distro/src/main/assembly/plugin-atlas.xml index d350612742..3d0a55cbc1 100644 --- a/distro/src/main/assembly/plugin-atlas.xml +++ b/distro/src/main/assembly/plugin-atlas.xml @@ -121,6 +121,8 @@ com.fasterxml.woodstox:woodstox-core:jar:${fasterxml.woodstox.version} org.codehaus.woodstox:stax2-api:jar:${codehaus.woodstox.stax2api.version} org.apache.commons:commons-configuration2:jar:${commons.configuration.version} + org.apache.commons:commons-lang3:jar:${commons.lang3.version} + org.apache.commons:commons-compress:jar:${commons.compress.version} From 9acade5ab22b1423814f74fa8d96789bdd48bb4f Mon Sep 17 00:00:00 2001 From: Madhan Neethiraj Date: Fri, 14 Jul 2023 20:04:09 -0700 Subject: [PATCH 07/46] RANGER-4320: createPrincipalsIfAbsent request parameter is not recognized by importPoliciesFromFile REST API --- .../java/org/apache/ranger/biz/RangerBizUtil.java | 10 ++-------- .../main/java/org/apache/ranger/rest/RoleREST.java | 6 ++---- .../java/org/apache/ranger/rest/ServiceREST.java | 12 ++++-------- .../ranger/security/context/RangerContextHolder.java | 12 ++++++++++++ .../filter/RangerSecurityContextFormationFilter.java | 11 +---------- 5 files changed, 21 insertions(+), 30 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java index 7a7cc8137d..136a1309ba 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java @@ -1553,14 +1553,8 @@ public static boolean isBulkMode() { } public static boolean setBulkMode(boolean val) { - if(RangerContextHolder.getOpContext()!=null){ - RangerContextHolder.getOpContext().setBulkModeContext(val); - } - else { - RangerAdminOpContext opContext = new RangerAdminOpContext(); - opContext.setBulkModeContext(val); - RangerContextHolder.setOpContext(opContext); - } + RangerContextHolder.getOrCreateOpContext().setBulkModeContext(val); + return isBulkMode(); } diff --git a/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java b/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java index ca9f286b3d..4bfaa862c2 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/RoleREST.java @@ -70,7 +70,6 @@ import org.apache.ranger.plugin.util.RangerRESTUtils; import org.apache.ranger.plugin.util.RangerRoles; import org.apache.ranger.plugin.util.SearchFilter; -import org.apache.ranger.security.context.RangerAdminOpContext; import org.apache.ranger.security.context.RangerContextHolder; import org.apache.ranger.service.RangerRoleService; import org.apache.ranger.service.XUserService; @@ -429,9 +428,8 @@ public RESTResponse importRolesFromFile(@Context HttpServletRequest request, } RESTResponse ret = new RESTResponse(); - RangerAdminOpContext opContext = new RangerAdminOpContext(); - opContext.setBulkModeContext(true); - RangerContextHolder.setOpContext(opContext); + RangerContextHolder.getOrCreateOpContext().setBulkModeContext(true); + String metaDataInfo = null; List trxLogListError = new ArrayList(); XXTrxLog xxTrxLogError = new XXTrxLog(); diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java index a307293eb4..d2d76733ea 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceREST.java @@ -60,7 +60,6 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.RandomStringUtils; import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang3.math.NumberUtils; import org.apache.ranger.admin.client.datatype.RESTResponse; import org.apache.ranger.authorization.hadoop.config.RangerAdminConfig; import org.apache.ranger.authorization.utils.StringUtil; @@ -128,7 +127,6 @@ import org.apache.ranger.plugin.util.SearchFilter; import org.apache.ranger.plugin.util.ServicePolicies; import org.apache.ranger.security.context.RangerAPIList; -import org.apache.ranger.security.context.RangerAdminOpContext; import org.apache.ranger.security.context.RangerContextHolder; import org.apache.ranger.security.web.filter.RangerCSRFPreventionFilter; import org.apache.ranger.service.RangerPluginInfoService; @@ -2264,9 +2262,9 @@ public void importPoliciesFromFile( if (LOG.isDebugEnabled()) { LOG.debug("==> ServiceREST.importPoliciesFromFile()"); } - RangerAdminOpContext opContext = new RangerAdminOpContext(); - opContext.setBulkModeContext(true); - RangerContextHolder.setOpContext(opContext); + + RangerContextHolder.getOrCreateOpContext().setBulkModeContext(true); + RangerPerfTracer perf = null; String metaDataInfo = null; List trxLogListError = new ArrayList(); @@ -4519,9 +4517,7 @@ private String deleteServiceById(Long id) { LOG.debug("==> ServiceREST.deleteServiceById( " + id + ")"); } - RangerAdminOpContext opContext = new RangerAdminOpContext(); - opContext.setBulkModeContext(true); - RangerContextHolder.setOpContext(opContext); + RangerContextHolder.getOrCreateOpContext().setBulkModeContext(true); RangerPerfTracer perf = null; String deletedServiceName = null; diff --git a/security-admin/src/main/java/org/apache/ranger/security/context/RangerContextHolder.java b/security-admin/src/main/java/org/apache/ranger/security/context/RangerContextHolder.java index e42dc2406d..d54eb556f9 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/context/RangerContextHolder.java +++ b/security-admin/src/main/java/org/apache/ranger/security/context/RangerContextHolder.java @@ -48,6 +48,18 @@ public static RangerAdminOpContext getOpContext() { return operationContextThreadLocal.get(); } + public static RangerAdminOpContext getOrCreateOpContext() { + RangerAdminOpContext ret = operationContextThreadLocal.get(); + + if (ret == null) { + ret = new RangerAdminOpContext(); + + operationContextThreadLocal.set(ret); + } + + return ret; + } + public static void setOpContext(RangerAdminOpContext context) { operationContextThreadLocal.set(context); } diff --git a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java index fee1d5895f..c0a0b7e2dd 100644 --- a/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java +++ b/security-admin/src/main/java/org/apache/ranger/security/web/filter/RangerSecurityContextFormationFilter.java @@ -40,7 +40,6 @@ import org.apache.ranger.common.RequestContext; import org.apache.ranger.common.UserSessionBase; import org.apache.ranger.entity.XXAuthSession; -import org.apache.ranger.security.context.RangerAdminOpContext; import org.apache.ranger.security.context.RangerContextHolder; import org.apache.ranger.security.context.RangerSecurityContext; import org.apache.ranger.util.RestUtil; @@ -156,15 +155,7 @@ private void setupAdminOpContext(ServletRequest request) { Object attrCreatePrincipalsIfAbsent = request.getParameter("createPrincipalsIfAbsent"); if (attrCreatePrincipalsIfAbsent != null) { - RangerAdminOpContext opContext = RangerContextHolder.getOpContext(); - - if (opContext == null) { - opContext = new RangerAdminOpContext(); - - RangerContextHolder.setOpContext(opContext); - } - - opContext.setCreatePrincipalsIfAbsent(Boolean.parseBoolean(attrCreatePrincipalsIfAbsent.toString())); + RangerContextHolder.getOrCreateOpContext().setCreatePrincipalsIfAbsent(Boolean.parseBoolean(attrCreatePrincipalsIfAbsent.toString())); } } From 1fa02389ec99085c3ab644bc85c39dd04f54e461 Mon Sep 17 00:00:00 2001 From: szymonorz Date: Wed, 26 Jul 2023 00:10:36 +0200 Subject: [PATCH 08/46] RANGER-4329: fix for tagsync failure during startup - included guava and commons-logging libraries Signed-off-by: Madhan Neethiraj --- tagsync/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tagsync/pom.xml b/tagsync/pom.xml index ede5e838ac..1df1ccb649 100644 --- a/tagsync/pom.xml +++ b/tagsync/pom.xml @@ -90,6 +90,11 @@ commons-lang ${commons.lang.version} + + commons-logging + commons-logging + ${commons.logging.version} + org.apache.commons commons-text @@ -303,5 +308,10 @@ ranger-common-ha ${project.version} + + com.google.guava + guava + ${google.guava.version} + From 52f07b9446a4e250bc43c57c3c6b934b25c9d25f Mon Sep 17 00:00:00 2001 From: Madhan Neethiraj Date: Wed, 26 Jul 2023 14:24:31 -0700 Subject: [PATCH 09/46] RANGER-4332: updated AuditBatchQueue.log() to block instead of throwing 'Queue full' exception --- .../org/apache/ranger/audit/queue/AuditBatchQueue.java | 9 +++++++-- .../java/org/apache/ranger/audit/TestAuditQueue.java | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java index d9cd52b599..afa2879e2e 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java @@ -56,8 +56,13 @@ public AuditBatchQueue(AuditHandler consumer) { */ @Override public boolean log(AuditEventBase event) { - // Add to batchQueue. Block if full - queue.add(event); + try { + // Add to batchQueue. Block if full + queue.put(event); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + return true; } diff --git a/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java b/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java index d30854bef5..c62fdd89ad 100644 --- a/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java +++ b/security-admin/src/test/java/org/apache/ranger/audit/TestAuditQueue.java @@ -255,7 +255,7 @@ public void testAuditBatchQueueBySize() { int batchSize = messageToSend / 3; int expectedBatchSize = batchSize + (batchSize * 3 < messageToSend ? 1 : 0); - int queueSize = messageToSend * 2; + int queueSize = batchSize * 2; int intervalMS = messageToSend * 100; // Deliberately big interval Properties props = new Properties(); props.put(basePropName + "." + AuditQueue.PROP_BATCH_SIZE, "" From f004731e50abbd1b450e9e8eb97274d394071f43 Mon Sep 17 00:00:00 2001 From: Brijesh Bhalala Date: Wed, 26 Jul 2023 11:40:10 +0530 Subject: [PATCH 10/46] RANGER-4293: Addendum change for Long User/group/role name overflowing from delete confirmation dialog box Signed-off-by: Mehul Parikh --- .../UserGroupRoleListing/groups_details/GroupListing.jsx | 7 +++---- .../UserGroupRoleListing/role_details/RoleListing.jsx | 7 +++---- .../UserGroupRoleListing/users_details/UserListing.jsx | 7 +++---- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/groups_details/GroupListing.jsx b/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/groups_details/GroupListing.jsx index 991e27ea97..2ba0ca068d 100644 --- a/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/groups_details/GroupListing.jsx +++ b/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/groups_details/GroupListing.jsx @@ -626,15 +626,14 @@ function Groups() { - {" "} - Are you sure you want to delete group  + Are you sure you want to delete the  {selectedRows.current.length === 1 ? ( <> - "{selectedRows.current[0].original.name}" ? + "{selectedRows.current[0].original.name}" group ? ) : ( <> - "{selectedRows.current.length}" ? + selected {selectedRows.current.length} groups? )} diff --git a/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/role_details/RoleListing.jsx b/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/role_details/RoleListing.jsx index e6293877c3..a65fcbe5c1 100644 --- a/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/role_details/RoleListing.jsx +++ b/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/role_details/RoleListing.jsx @@ -406,15 +406,14 @@ function Roles() { - {" "} - Are you sure you want to delete group  + Are you sure you want to delete the  {selectedRows.current.length === 1 ? ( <> - "{selectedRows.current[0].original.name}" ? + "{selectedRows.current[0].original.name}" role ? ) : ( <> - "{selectedRows.current.length}" ? + selected {selectedRows.current.length} roles? )} diff --git a/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/users_details/UserListing.jsx b/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/users_details/UserListing.jsx index 95a21664d1..c3150fd65e 100644 --- a/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/users_details/UserListing.jsx +++ b/security-admin/src/main/webapp/react-webapp/src/views/UserGroupRoleListing/users_details/UserListing.jsx @@ -692,15 +692,14 @@ function Users() { - {" "} - Are you sure you want to delete user  + Are you sure you want to delete the  {selectedRows.current.length === 1 ? ( <> - "{selectedRows.current[0].original.name}" ? + "{selectedRows.current[0].original.name}" user ? ) : ( <> - "{selectedRows.current.length}" ? + selected {selectedRows.current.length} users? )} From f0b07d363fec2b6396961c99ae8f2df88e4ac4c6 Mon Sep 17 00:00:00 2001 From: Brijesh Bhalala Date: Fri, 14 Jul 2023 12:33:59 +0530 Subject: [PATCH 11/46] RANGER-4317: Error message displayed when resource lookup fails is not formatted properly Signed-off-by: Mehul Parikh --- .../src/main/webapp/react-webapp/src/styles/style.css | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/security-admin/src/main/webapp/react-webapp/src/styles/style.css b/security-admin/src/main/webapp/react-webapp/src/styles/style.css index 8b86453a4f..aaa54a380e 100644 --- a/security-admin/src/main/webapp/react-webapp/src/styles/style.css +++ b/security-admin/src/main/webapp/react-webapp/src/styles/style.css @@ -2535,6 +2535,15 @@ li.list-group-item:hover { border-color: #0062cc; box-shadow: 0 0 0 0.2rem rgb(38 143 255 / 50%); } +.Toastify__toast { + word-break: break-word; +} +.Toastify__toast-container { + width: 384px; +} +.Toastify__toast-icon { + width: 16px; +} .manage-service .dropdown-toggle::after { vertical-align: 0.125em; } From 786bdc82dd8ee5640a325e65b15be23a1744f36d Mon Sep 17 00:00:00 2001 From: Mugdha Varadkar Date: Wed, 26 Jul 2023 13:04:12 +0530 Subject: [PATCH 12/46] RANGER-4331: Fixes for search filter on Audits tabs Signed-off-by: Mehul Parikh --- .../webapp/react-webapp/src/utils/XAUtils.js | 11 +- .../src/views/AuditEvent/AccessLogs.jsx | 169 +++++++++--------- .../src/views/AuditEvent/AuditLayout.jsx | 38 +++- .../src/views/AuditEvent/PluginStatusLogs.jsx | 81 ++------- .../src/views/AuditEvent/PluginsLog.jsx | 14 +- .../ServiceManager/ServiceDefinitions.jsx | 4 +- 6 files changed, 150 insertions(+), 167 deletions(-) diff --git a/security-admin/src/main/webapp/react-webapp/src/utils/XAUtils.js b/security-admin/src/main/webapp/react-webapp/src/utils/XAUtils.js index 30fc4225be..74b213a9d9 100644 --- a/security-admin/src/main/webapp/react-webapp/src/utils/XAUtils.js +++ b/security-admin/src/main/webapp/react-webapp/src/utils/XAUtils.js @@ -1162,10 +1162,11 @@ export const fetchSearchFilterParams = ( // Get search filter params from localStorage if (isEmpty(searchFilterParam)) { - const localStorageParams = - !isEmpty(localStorage.getItem(auditTabName)) && - JSON.parse(localStorage.getItem(auditTabName)); - if (!isNull(localStorageParams) && !isEmpty(localStorageParams)) { + if (!isNull(localStorage.getItem(auditTabName))) { + const localStorageParams = + !isEmpty(localStorage.getItem(auditTabName)) && + JSON.parse(localStorage.getItem(auditTabName)); + for (const localParam in localStorageParams) { let searchFilterObj = find(searchFilterOptions, { urlLabel: localParam @@ -1187,7 +1188,7 @@ export const fetchSearchFilterParams = ( category: category, value: value }); - searchParam[localParam] = value; + searchParam[localParam] = localStorageParams[localParam]; } } } diff --git a/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AccessLogs.jsx b/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AccessLogs.jsx index b30c6f32f8..630e516376 100644 --- a/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AccessLogs.jsx +++ b/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AccessLogs.jsx @@ -18,7 +18,7 @@ */ import React, { useState, useCallback, useEffect, useRef } from "react"; -import { useSearchParams } from "react-router-dom"; +import { useSearchParams, useOutletContext } from "react-router-dom"; import { Badge, Button, Row, Col, Table, Modal } from "react-bootstrap"; import XATableLayout from "Components/XATableLayout"; import dateFormat from "dateformat"; @@ -40,7 +40,8 @@ import { toString, toUpper, has, - filter + filter, + isNull } from "lodash"; import { toast } from "react-toastify"; import { Link } from "react-router-dom"; @@ -62,12 +63,15 @@ import { ServiceRequestDataRangerAcl, ServiceRequestDataHadoopAcl } from "../../utils/XAEnums"; +import { getServiceDef } from "../../utils/appState"; function Access() { + const context = useOutletContext(); + const services = context.services; + const servicesAvailable = context.servicesAvailable; const isKMSRole = isKeyAdmin() || isKMSAuditor(); + const [searchParams, setSearchParams] = useSearchParams(); const [accessListingData, setAccessLogs] = useState([]); - const [serviceDefs, setServiceDefs] = useState([]); - const [services, setServices] = useState([]); const [zones, setZones] = useState([]); const [loader, setLoader] = useState(true); const [pageCount, setPageCount] = React.useState(0); @@ -77,73 +81,88 @@ function Access() { const [policyviewmodal, setPolicyViewModal] = useState(false); const [policyParamsData, setPolicyParamsData] = useState(null); const [rowdata, setRowData] = useState([]); - const [checked, setChecked] = useState(false); + const [checked, setChecked] = useState(() => { + let urlParam = Object.fromEntries([...searchParams]); + if (urlParam?.excludeServiceUser) { + return urlParam.excludeServiceUser == "true" ? true : false; + } else { + return localStorage?.excludeServiceUser == "true" ? true : false; + } + }); const [currentPage, setCurrentPage] = useState(1); const fetchIdRef = useRef(0); const [contentLoader, setContentLoader] = useState(true); const [searchFilterParams, setSearchFilterParams] = useState([]); - const [searchParams, setSearchParams] = useSearchParams(); const [defaultSearchFilterParams, setDefaultSearchFilterParams] = useState( [] ); const [resetPage, setResetpage] = useState({ page: 0 }); const [policyDetails, setPolicyDetails] = useState({}); + const { allServiceDefs } = getServiceDef(); useEffect(() => { - if (isEmpty(serviceDefs)) { - fetchServiceDefs(), fetchServices(); + if (!isKMSRole) { + fetchZones(); + } - if (!isKMSRole) { - fetchZones(); - } + let currentDate = moment(moment()).format("MM/DD/YYYY"); + let { searchFilterParam, defaultSearchFilterParam, searchParam } = + fetchSearchFilterParams("bigData", searchParams, searchFilterOptions); + + if ( + !has(searchFilterParam, "startDate") && + !has(searchFilterParam, "endDate") + ) { + searchParam["startDate"] = currentDate; + searchFilterParam["startDate"] = currentDate; + defaultSearchFilterParam.push({ + category: "startDate", + value: currentDate + }); } - if (!isEmpty(serviceDefs)) { - let currentDate = moment(moment()).format("MM/DD/YYYY"); + // Updating the states for search params, search filter, default search filter and localStorage + if (localStorage?.excludeServiceUser) { + searchParam["excludeServiceUser"] = checked; + } + setSearchParams(searchParam); + setSearchFilterParams(searchFilterParam); + setDefaultSearchFilterParams(defaultSearchFilterParam); + localStorage.setItem("bigData", JSON.stringify(searchParam)); + }, []); + + useEffect(() => { + if (servicesAvailable !== null) { let { searchFilterParam, defaultSearchFilterParam, searchParam } = fetchSearchFilterParams("bigData", searchParams, searchFilterOptions); + // Updating the states for search params, search filter, default search filter and localStorage + if (localStorage?.excludeServiceUser || searchParam?.excludeServiceUser) { + if (searchParam?.excludeServiceUser) { + setChecked(searchParam?.excludeServiceUser == "true" ? true : false); + localStorage.setItem( + "excludeServiceUser", + searchParam?.excludeServiceUser + ); + } + searchParam["excludeServiceUser"] = localStorage?.excludeServiceUser; + } + setSearchParams(searchParam); if ( - !has(searchFilterParam, "startDate") && - !has(searchFilterParam, "endDate") + JSON.stringify(searchFilterParams) !== JSON.stringify(searchFilterParam) ) { - searchParam["startDate"] = currentDate; - searchFilterParam["startDate"] = currentDate; - defaultSearchFilterParam.push({ - category: "startDate", - value: currentDate - }); + setSearchFilterParams(searchFilterParam); } - - // Updating the states for search params, search filter, default search filter and localStorage - setSearchParams(searchParam); - setSearchFilterParams(searchFilterParam); setDefaultSearchFilterParams(defaultSearchFilterParam); localStorage.setItem("bigData", JSON.stringify(searchParam)); setContentLoader(false); } - }, [serviceDefs]); - - useEffect(() => { - let { searchFilterParam, defaultSearchFilterParam, searchParam } = - fetchSearchFilterParams("bigData", searchParams, searchFilterOptions); - - // Updating the states for search params, search filter, default search filter and localStorage - setSearchParams(searchParam); - if ( - JSON.stringify(searchFilterParams) !== JSON.stringify(searchFilterParam) - ) { - setSearchFilterParams(searchFilterParam); - } - setDefaultSearchFilterParams(defaultSearchFilterParam); - localStorage.setItem("bigData", JSON.stringify(searchParam)); - setContentLoader(false); - }, [searchParams]); + }, [searchParams, servicesAvailable]); const fetchAccessLogsInfo = useCallback( async ({ pageSize, pageIndex, sortBy, gotoPage }) => { setLoader(true); - if (!isEmpty(serviceDefs)) { + if (servicesAvailable !== null) { let logsResp = []; let logs = []; let totalCount = 0; @@ -152,7 +171,15 @@ function Access() { if (fetchId === fetchIdRef.current) { params["pageSize"] = pageSize; params["startIndex"] = pageIndex * pageSize; - params["excludeServiceUser"] = checked ? true : false; + if (Object.fromEntries([...searchParams])?.excludeServiceUser) { + params["excludeServiceUser"] = + Object.fromEntries([...searchParams])?.excludeServiceUser == + "true" + ? true + : false; + } else { + params["excludeServiceUser"] = checked; + } if (sortBy.length > 0) { params["sortBy"] = getTableSortBy(sortBy); params["sortType"] = getTableSortType(sortBy); @@ -178,41 +205,9 @@ function Access() { } } }, - [updateTable, searchFilterParams] + [updateTable, checked, searchFilterParams, servicesAvailable] ); - const fetchServiceDefs = async () => { - let serviceDefsResp = []; - try { - serviceDefsResp = await fetchApi({ - url: "plugins/definitions" - }); - } catch (error) { - console.error( - `Error occurred while fetching Service Definitions or CSRF headers! ${error}` - ); - } - - setServiceDefs(serviceDefsResp.data.serviceDefs); - setContentLoader(false); - }; - - const fetchServices = async () => { - let servicesResp = []; - try { - servicesResp = await fetchApi({ - url: "plugins/services" - }); - } catch (error) { - console.error( - `Error occurred while fetching Services or CSRF headers! ${error}` - ); - } - - setServices(servicesResp.data.services); - setContentLoader(false); - }; - const fetchZones = async () => { let zonesResp; try { @@ -224,16 +219,18 @@ function Access() { } setZones(sortBy(zonesResp.data.securityZones, ["name"])); - setContentLoader(false); }; - const toggleChange = () => { + const toggleChange = (chkVal) => { let currentParams = Object.fromEntries([...searchParams]); - currentParams["excludeServiceUser"] = !checked; - localStorage.setItem("excludeServiceUser", JSON.stringify(!checked)); + currentParams["excludeServiceUser"] = chkVal?.target?.checked; + localStorage.setItem( + "excludeServiceUser", + JSON.stringify(chkVal?.target?.checked) + ); setSearchParams(currentParams); setAccessLogs([]); - setChecked(!checked); + setChecked(chkVal?.target?.checked); setLoader(true); setUpdateTable(moment.now()); }; @@ -693,7 +690,7 @@ function Access() { const getServiceDefType = () => { let serviceDefType = []; - serviceDefType = filter(serviceDefs, function (serviceDef) { + serviceDefType = filter(allServiceDefs, function (serviceDef) { return serviceDef.name !== "tag"; }); @@ -935,10 +932,8 @@ function Access() { { - toggleChange(); - }} + checked={checked} + onChange={toggleChange} data-id="serviceUsersExclude" data-cy="serviceUsersExclude" /> diff --git a/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AuditLayout.jsx b/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AuditLayout.jsx index a51b7ac2fe..0537496f33 100644 --- a/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AuditLayout.jsx +++ b/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/AuditLayout.jsx @@ -21,12 +21,15 @@ import React, { Component } from "react"; import { Tab, Tabs } from "react-bootstrap"; import withRouter from "Hooks/withRouter"; import { Outlet } from "react-router-dom"; +import { fetchApi } from "Utils/fetchAPI"; class AuditLayout extends Component { constructor(props) { super(props); this.state = { - activeKey: this.activeTab() + activeKey: this.activeTab(), + services: null, + servicesAvailable: null }; } @@ -43,6 +46,32 @@ class AuditLayout extends Component { } } + componentDidMount() { + this.fetchServices(); + } + + fetchServices = async () => { + let servicesResp = []; + try { + const response = await fetchApi({ + url: "plugins/services" + }); + servicesResp = response?.data?.services || []; + } catch (error) { + console.error( + `Error occurred while fetching Services or CSRF headers! ${error}` + ); + } + + this.setState({ + services: servicesResp, + servicesAvailable: + servicesResp.length === 0 + ? "SERVICES_NOT_AVAILABLE" + : "SERVICES_AVAILABLE" + }); + }; + activeTab = () => { let activeTabVal; if (this.props?.location?.pathname) { @@ -85,7 +114,12 @@ class AuditLayout extends Component { - + ); } diff --git a/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/PluginStatusLogs.jsx b/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/PluginStatusLogs.jsx index b1bcca6562..ea0e550a7c 100644 --- a/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/PluginStatusLogs.jsx +++ b/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/PluginStatusLogs.jsx @@ -18,7 +18,7 @@ */ import React, { useState, useCallback, useRef, useEffect } from "react"; -import { useSearchParams } from "react-router-dom"; +import { useSearchParams, useOutletContext } from "react-router-dom"; import { Row, Col } from "react-bootstrap"; import XATableLayout from "Components/XATableLayout"; import { AuditFilterEntries } from "Components/CommonComponents"; @@ -40,16 +40,18 @@ import { import StructuredFilter from "../../components/structured-filter/react-typeahead/tokenizer"; import { fetchApi } from "Utils/fetchAPI"; import { isEmpty, isUndefined, map, sortBy, toUpper, filter } from "lodash"; +import { getServiceDef } from "../../utils/appState"; function Plugin_Status() { + const context = useOutletContext(); + const services = context.services; + const servicesAvailable = context.servicesAvailable; const [pluginStatusListingData, setPluginStatusLogs] = useState([]); const [loader, setLoader] = useState(true); const [pageCount, setPageCount] = React.useState(0); const [entries, setEntries] = useState([]); const [updateTable, setUpdateTable] = useState(moment.now()); const fetchIdRef = useRef(0); - const [serviceDefs, setServiceDefs] = useState([]); - const [services, setServices] = useState([]); const [contentLoader, setContentLoader] = useState(true); const [searchFilterParams, setSearchFilterParams] = useState([]); const [searchParams, setSearchParams] = useSearchParams(); @@ -58,13 +60,10 @@ function Plugin_Status() { ); const [resetPage, setResetpage] = useState({ page: null }); const isKMSRole = isKeyAdmin() || isKMSAuditor(); + const { allServiceDefs } = getServiceDef(); useEffect(() => { - if (isEmpty(serviceDefs)) { - fetchServiceDefs(), fetchServices(); - } - - if (!isEmpty(serviceDefs)) { + if (servicesAvailable !== null) { let { searchFilterParam, defaultSearchFilterParam, searchParam } = fetchSearchFilterParams( "pluginStatus", @@ -74,69 +73,21 @@ function Plugin_Status() { // Updating the states for search params, search filter, default search filter and localStorage setSearchParams(searchParam); - setSearchFilterParams(searchFilterParam); + if ( + JSON.stringify(searchFilterParams) !== JSON.stringify(searchFilterParam) + ) { + setSearchFilterParams(searchFilterParam); + } setDefaultSearchFilterParams(defaultSearchFilterParam); localStorage.setItem("pluginStatus", JSON.stringify(searchParam)); setContentLoader(false); } - }, [serviceDefs]); - - useEffect(() => { - let { searchFilterParam, defaultSearchFilterParam, searchParam } = - fetchSearchFilterParams( - "pluginStatus", - searchParams, - searchFilterOptions - ); - - // Updating the states for search params, search filter, default search filter and localStorage - setSearchParams(searchParam); - if ( - JSON.stringify(searchFilterParams) !== JSON.stringify(searchFilterParam) - ) { - setSearchFilterParams(searchFilterParam); - } - setDefaultSearchFilterParams(defaultSearchFilterParam); - localStorage.setItem("pluginStatus", JSON.stringify(searchParam)); - setContentLoader(false); - }, [searchParams]); - - const fetchServiceDefs = async () => { - setLoader(true); - let serviceDefsResp = []; - try { - serviceDefsResp = await fetchApi({ - url: "plugins/definitions" - }); - } catch (error) { - console.error( - `Error occurred while fetching Service Definitions or CSRF headers! ${error}` - ); - } - - setServiceDefs(serviceDefsResp.data.serviceDefs); - setLoader(false); - }; - - const fetchServices = async () => { - let servicesResp = []; - try { - servicesResp = await fetchApi({ - url: "plugins/services" - }); - } catch (error) { - console.error( - `Error occurred while fetching Services or CSRF headers! ${error}` - ); - } - setServices(servicesResp.data.services); - setContentLoader(false); - }; + }, [searchParams, servicesAvailable]); const fetchPluginStatusInfo = useCallback( async ({ pageSize, pageIndex, gotoPage }) => { setLoader(true); - if (!isEmpty(serviceDefs)) { + if (servicesAvailable !== null) { let logsResp = []; let logs = []; let totalCount = 0; @@ -166,7 +117,7 @@ function Plugin_Status() { } } }, - [updateTable, searchFilterParams] + [updateTable, searchFilterParams, servicesAvailable] ); const isDateDifferenceMoreThanHr = (date1, date2) => { @@ -535,7 +486,7 @@ function Plugin_Status() { const getServiceDefType = () => { let serviceDefType = []; - serviceDefType = map(serviceDefs, function (serviceDef) { + serviceDefType = map(allServiceDefs, function (serviceDef) { return { label: toUpper(serviceDef.displayName), value: serviceDef.name diff --git a/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/PluginsLog.jsx b/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/PluginsLog.jsx index 37ae4e1c89..55caa90bd8 100644 --- a/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/PluginsLog.jsx +++ b/security-admin/src/main/webapp/react-webapp/src/views/AuditEvent/PluginsLog.jsx @@ -18,7 +18,7 @@ */ import React, { useState, useCallback, useRef, useEffect } from "react"; -import { useSearchParams } from "react-router-dom"; +import { useSearchParams, useOutletContext } from "react-router-dom"; import { Badge, Row, Col } from "react-bootstrap"; import XATableLayout from "Components/XATableLayout"; import { AuditFilterEntries } from "Components/CommonComponents"; @@ -39,13 +39,15 @@ import { import { Loader } from "../../components/CommonComponents"; function Plugins() { + const context = useOutletContext(); + const services = context.services; const [pluginsListingData, setPluginsLogs] = useState([]); const [loader, setLoader] = useState(true); const [pageCount, setPageCount] = React.useState(0); const [entries, setEntries] = useState([]); const [updateTable, setUpdateTable] = useState(moment.now()); const fetchIdRef = useRef(0); - const [services, setServices] = useState([]); + //const [services, setServices] = useState([]); const [contentLoader, setContentLoader] = useState(true); const [searchFilterParams, setSearchFilterParams] = useState([]); const [searchParams, setSearchParams] = useSearchParams(); @@ -56,8 +58,6 @@ function Plugins() { const isKMSRole = isKeyAdmin() || isKMSAuditor(); useEffect(() => { - fetchServices(); - let { searchFilterParam, defaultSearchFilterParam, searchParam } = fetchSearchFilterParams("agent", searchParams, searchFilterOptions); @@ -73,7 +73,7 @@ function Plugins() { setContentLoader(false); }, [searchParams]); - const fetchServices = async () => { + /* const fetchServices = async () => { let servicesResp = []; try { servicesResp = await fetchApi({ @@ -87,7 +87,7 @@ function Plugins() { setServices(servicesResp.data.services); setLoader(false); - }; + }; */ const fetchPluginsInfo = useCallback( async ({ pageSize, pageIndex, sortBy, gotoPage }) => { @@ -265,7 +265,7 @@ function Plugins() { }); return sortBy(servicesName, "name")?.map((service) => ({ - label: service.displayName, + label: service.name, value: service.name })); }; diff --git a/security-admin/src/main/webapp/react-webapp/src/views/ServiceManager/ServiceDefinitions.jsx b/security-admin/src/main/webapp/react-webapp/src/views/ServiceManager/ServiceDefinitions.jsx index db2710bb96..aeb88161ba 100644 --- a/security-admin/src/main/webapp/react-webapp/src/views/ServiceManager/ServiceDefinitions.jsx +++ b/security-admin/src/main/webapp/react-webapp/src/views/ServiceManager/ServiceDefinitions.jsx @@ -72,7 +72,9 @@ class ServiceDefinitions extends Component { initialFetchResp = async () => { await this.fetchServices(); - await this.fetchZones(); + if (!this.state.isKMSRole) { + await this.fetchZones(); + } }; showExportModal = () => { From f46357f1647aca98f10bb2e5824bde9955ad2fb2 Mon Sep 17 00:00:00 2001 From: Pradeep AgrawaL Date: Tue, 1 Aug 2023 15:31:37 +0530 Subject: [PATCH 13/46] RANGER-4337: Upgrade spring-framework and spring-security --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index d4f6262026..c7a8606502 100644 --- a/pom.xml +++ b/pom.xml @@ -199,9 +199,9 @@ 1.7.32 8.11.2 2.4.1 - 5.7.8 - 5.3.27 - 5.3.27 + 5.7.10 + 5.3.29 + 5.3.29 1.99.7 1.2.4 1.19 From 3114baac5e4aa0f461152de74354ab0ab2d9e258 Mon Sep 17 00:00:00 2001 From: Madhan Neethiraj Date: Thu, 27 Jul 2023 01:15:34 -0700 Subject: [PATCH 14/46] RANGER-4336: added configurations to enable status logging in audit framework --- .../audit/provider/BaseAuditHandler.java | 36 +++++++++++++++---- .../ranger/audit/queue/AuditAsyncQueue.java | 18 ++++++++++ .../org/apache/ranger/audit/TestConsumer.java | 26 +++++++------- 3 files changed, 61 insertions(+), 19 deletions(-) diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java index 8511ce9cbb..68c33c90db 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java @@ -38,6 +38,11 @@ public abstract class BaseAuditHandler implements AuditHandler { static final String AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP = "xasecure.audit.log.failure.report.min.interval.ms"; + static final String AUDIT_LOG_STATUS_LOG_ENABLED = "xasecure.audit.log.status.log.enabled"; + static final String AUDIT_LOG_STATUS_LOG_INTERVAL_SEC = "xasecure.audit.log.status.log.interval.sec"; + static final boolean DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED = false; + static final long DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC = 5 * 60; // 5 minutes + public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE = "xasecure.policymgr.clientssl.keystore"; public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_TYPE = "xasecure.policymgr.clientssl.keystore.type"; public static final String RANGER_POLICYMGR_CLIENT_KEY_FILE_CREDENTIAL = "xasecure.policymgr.clientssl.keystore.credential.file"; @@ -90,8 +95,10 @@ public abstract class BaseAuditHandler implements AuditHandler { long lastStashedCount = 0; long lastDeferredCount = 0; - long lastStatusLogTime = System.currentTimeMillis(); - long statusLogIntervalMS = 1 * 60 * 1000; + boolean statusLogEnabled = DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED; + long statusLogIntervalMS = DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC * 1000; + long lastStatusLogTime = System.currentTimeMillis(); + long nextStatusLogTime = lastStatusLogTime + statusLogIntervalMS; protected Properties props = null; protected Map configProps = new HashMap(); @@ -138,6 +145,19 @@ public void init(Properties props, String basePropertyName) { mLogFailureReportMinIntervalInMs = MiscUtil.getIntProperty(props, AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP, 60 * 1000); + boolean globalStatusLogEnabled = MiscUtil.getBooleanProperty(props, AUDIT_LOG_STATUS_LOG_ENABLED, DEFAULT_AUDIT_LOG_STATUS_LOG_ENABLED); + long globalStatusLogIntervalSec = MiscUtil.getLongProperty(props, AUDIT_LOG_STATUS_LOG_INTERVAL_SEC, DEFAULT_AUDIT_LOG_STATUS_LOG_INTERVAL_SEC); + + statusLogEnabled = MiscUtil.getBooleanProperty(props, basePropertyName + ".status.log.enabled", globalStatusLogEnabled); + statusLogIntervalMS = MiscUtil.getLongProperty(props, basePropertyName + ".status.log.interval.sec", globalStatusLogIntervalSec) * 1000; + + nextStatusLogTime = lastStatusLogTime + statusLogIntervalMS; + + LOG.info(AUDIT_LOG_STATUS_LOG_ENABLED + "=" + globalStatusLogEnabled); + LOG.info(AUDIT_LOG_STATUS_LOG_INTERVAL_SEC + "=" + globalStatusLogIntervalSec); + LOG.info(basePropertyName + ".status.log.enabled=" + statusLogEnabled); + LOG.info(basePropertyName + ".status.log.interval.sec=" + (statusLogIntervalMS / 1000)); + String configPropsNamePrefix = propPrefix + "." + PROP_CONFIG + "."; for (Object propNameObj : props.keySet()) { String propName = propNameObj.toString(); @@ -275,9 +295,10 @@ public long getLastDeferredCount() { return lastDeferredCount; } + public boolean isStatusLogEnabled() { return statusLogEnabled; } + public void logStatusIfRequired() { - long currTime = System.currentTimeMillis(); - if ((currTime - lastStatusLogTime) > statusLogIntervalMS) { + if (System.currentTimeMillis() > nextStatusLogTime) { logStatus(); } } @@ -285,9 +306,10 @@ public void logStatusIfRequired() { public void logStatus() { try { long currTime = System.currentTimeMillis(); - long diffTime = currTime - lastStatusLogTime; + lastStatusLogTime = currTime; + nextStatusLogTime = currTime + statusLogIntervalMS; long diffCount = totalCount - lastIntervalCount; long diffSuccess = totalSuccessCount - lastIntervalSuccessCount; @@ -306,7 +328,7 @@ public void logStatus() { lastStashedCount = totalStashedCount; lastDeferredCount = totalDeferredCount; - if (LOG.isDebugEnabled()) { + if (statusLogEnabled) { String finalPath = ""; String tFinalPath = getFinalPath(); if (!getName().equals(tFinalPath)) { @@ -336,7 +358,7 @@ public void logStatus() { : "") + (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount) : ""); - LOG.debug(msg); + LOG.info(msg); } } catch (Throwable t) { LOG.error("Error while printing stats. auditProvider=" + getName()); diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java index 68527d37da..b226b4e201 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditAsyncQueue.java @@ -56,8 +56,13 @@ public AuditAsyncQueue(AuditHandler consumer) { */ @Override public boolean log(AuditEventBase event) { + logStatusIfRequired(); + + addTotalCount(1); + // Add to the queue and return ASAP if (queue.size() >= getMaxQueueSize()) { + addFailedCount(1); return false; } queue.add(event); @@ -134,6 +139,17 @@ public void run() { } } + @Override + public void logStatus() { + super.logStatus(); + + if (isStatusLogEnabled()) { + logger.info("AuditAsyncQueue.log(name={}): totalCount={}, currentQueueLength={}", getName(), getTotalCount(), queue.size()); + } + } + + public int size() { return queue.size(); } + public void runLogAudit() { while (true) { try { @@ -150,6 +166,8 @@ public void runLogAudit() { eventList.add(event); queue.drainTo(eventList, MAX_DRAIN - 1); consumer.log(eventList); + + logStatusIfRequired(); } } catch (InterruptedException e) { logger.info("Caught exception in consumer thread. Shutdown might be in progress"); diff --git a/security-admin/src/test/java/org/apache/ranger/audit/TestConsumer.java b/security-admin/src/test/java/org/apache/ranger/audit/TestConsumer.java index 579485663b..09386d2318 100644 --- a/security-admin/src/test/java/org/apache/ranger/audit/TestConsumer.java +++ b/security-admin/src/test/java/org/apache/ranger/audit/TestConsumer.java @@ -19,9 +19,7 @@ package org.apache.ranger.audit; -import java.util.ArrayList; import java.util.Collection; -import java.util.List; import java.util.Properties; import org.apache.ranger.audit.destination.AuditDestination; @@ -37,11 +35,11 @@ public class TestConsumer extends AuditDestination { int countTotal = 0; int sumTotal = 0; int batchCount = 0; + AuthzAuditEvent lastEvent = null; + AuthzAuditEvent lastOutOfSeqEvent = null; String providerName = getClass().getName(); boolean isDown = false; - List eventList = new ArrayList(); - /* * (non-Javadoc) * @@ -58,7 +56,8 @@ public boolean log(AuditEventBase event) { AuthzAuditEvent azEvent = (AuthzAuditEvent) event; sumTotal += azEvent.getEventCount(); logger.info("EVENT:" + event); - eventList.add(azEvent); + + processEvent(azEvent); } return true; } @@ -85,7 +84,7 @@ public boolean logJSON(String jsonStr) { AuthzAuditEvent.class); sumTotal += event.getEventCount(); logger.info("JSON:" + jsonStr); - eventList.add(event); + processEvent(event); return true; } @@ -198,13 +197,16 @@ public String getName() { // Local methods public AuthzAuditEvent isInSequence() { - long lastSeq = -1; - for (AuthzAuditEvent event : eventList) { - if (event.getSeqNum() <= lastSeq) { - return event; + return lastOutOfSeqEvent; + } + + private void processEvent(AuthzAuditEvent azEvent) { + if (lastEvent == null) { + lastEvent = azEvent; + } else if (lastOutOfSeqEvent == null) { + if (azEvent.getSeqNum() <= lastEvent.getSeqNum()) { + lastOutOfSeqEvent = azEvent; } - lastSeq = event.getSeqNum(); } - return null; } } From f1f5c02e29e50fb175c1dcb7756638e58f65c207 Mon Sep 17 00:00:00 2001 From: Dineshkumar Yadav Date: Fri, 4 Aug 2023 08:55:17 +0530 Subject: [PATCH 15/46] Revert "RANGER-4300: HBase shell revoke command failed with 'HTTP 400 Error: processSecureRevokeRequest processing failed'" This reverts commit ff38d0b3ee474c9fa9332311ed31b56e53e858dd. --- .../apache/ranger/rest/ServiceRESTUtil.java | 57 ------------------- 1 file changed, 57 deletions(-) diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceRESTUtil.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceRESTUtil.java index 4385573b69..60e34c0c7d 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceRESTUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceRESTUtil.java @@ -95,31 +95,6 @@ static public boolean processRevokeRequest(RangerPolicy existingRangerPolicy, Gr // remove all existing privileges for users and groups if (revokeRequest.getReplaceExistingPermissions()) { policyUpdated = removeUsersGroupsAndRolesFromPolicy(existingRangerPolicy, revokeRequest.getUsers(), revokeRequest.getGroups(), revokeRequest.getRoles()); - - // handling hbase shell revoke access for users - if (CollectionUtils.isNotEmpty(revokeRequest.getUsers()) || CollectionUtils.isNotEmpty(revokeRequest.getGroups()) - || CollectionUtils.isNotEmpty(revokeRequest.getRoles())) { - RangerPolicy appliedPolicy = new RangerPolicy(); - RangerPolicy.RangerPolicyItem deniedRangerPolicyItem = new RangerPolicy.RangerPolicyItem(); - - deniedRangerPolicyItem.setDelegateAdmin(false); - deniedRangerPolicyItem.getUsers().addAll(revokeRequest.getUsers()); - deniedRangerPolicyItem.getGroups().addAll(revokeRequest.getGroups()); - deniedRangerPolicyItem.getRoles().addAll(revokeRequest.getRoles()); - - List deniedRangerPolicyItemAccess = new ArrayList(); - - Set deniedPolicyItemAccessType = revokeRequest.getAccessTypes(); - for (String accessType : deniedPolicyItemAccessType) { - deniedRangerPolicyItemAccess.add(new RangerPolicy.RangerPolicyItemAccess(accessType, true)); - } - - deniedRangerPolicyItem.setAccesses(deniedRangerPolicyItemAccess); - - appliedPolicy.getDenyPolicyItems().add(deniedRangerPolicyItem); - processApplyPolicy(existingRangerPolicy, appliedPolicy); - policyUpdated = true; - } } else { //Build a policy and set up policyItem in it to mimic revoke request RangerPolicy appliedRangerPolicy = new RangerPolicy(); @@ -1086,7 +1061,6 @@ static private boolean removeUsersGroupsAndRolesFromPolicy(RangerPolicy policy, boolean policyUpdated = false; List policyItems = policy.getPolicyItems(); - List denyPolicyItems = policy.getDenyPolicyItems(); int numOfItems = policyItems.size(); @@ -1120,37 +1094,6 @@ static private boolean removeUsersGroupsAndRolesFromPolicy(RangerPolicy policy, } } - for (int i = 0; i < denyPolicyItems.size(); i++) { - RangerPolicy.RangerPolicyItem policyItem = denyPolicyItems.get(i); - - if (CollectionUtils.containsAny(policyItem.getUsers(), users)) { - policyItem.getUsers().removeAll(users); - - policyUpdated = true; - } - - if (CollectionUtils.containsAny(policyItem.getGroups(), groups)) { - policyItem.getGroups().removeAll(groups); - - policyUpdated = true; - } - - if (CollectionUtils.containsAny(policyItem.getRoles(), roles)) { - policyItem.getRoles().removeAll(roles); - - policyUpdated = true; - } - - if (CollectionUtils.isEmpty(policyItem.getUsers()) && CollectionUtils.isEmpty(policyItem.getGroups()) - && CollectionUtils.isEmpty(policyItem.getRoles())) { - denyPolicyItems.remove(i); - numOfItems--; - i--; - - policyUpdated = true; - } - } - return policyUpdated; } From cbd187f4d5c8404ecc77c4b9f0335c077029fe0c Mon Sep 17 00:00:00 2001 From: princeap173 Date: Wed, 2 Aug 2023 12:43:32 +0530 Subject: [PATCH 16/46] RANGER-4339: Correct spellings in React JS forms Signed-off-by: Kishor Gollapalliwar --- .../main/webapp/react-webapp/src/components/Editable.jsx | 2 +- .../src/views/AuditEvent/AdminLogs/PolicyLogs.jsx | 6 +++++- .../src/views/AuditEvent/AdminLogs/PolicyViewDetails.jsx | 6 +++--- .../src/views/PolicyListing/PolicyPermissionItem.jsx | 4 ++-- .../react-webapp/src/views/SecurityZone/ZoneListing.jsx | 2 +- .../webapp/react-webapp/src/views/SideBar/SideBarBody.jsx | 8 ++++---- .../views/UserGroupRoleListing/UserGroupRoleListing.jsx | 2 +- .../views/UserGroupRoleListing/role_details/RoleForm.jsx | 4 ++-- .../UserGroupRoleListing/role_details/RoleListing.jsx | 2 +- 9 files changed, 20 insertions(+), 16 deletions(-) diff --git a/security-admin/src/main/webapp/react-webapp/src/components/Editable.jsx b/security-admin/src/main/webapp/react-webapp/src/components/Editable.jsx index ea22b32adc..7dd32ebaa5 100644 --- a/security-admin/src/main/webapp/react-webapp/src/components/Editable.jsx +++ b/security-admin/src/main/webapp/react-webapp/src/components/Editable.jsx @@ -442,7 +442,7 @@ const Editable = (props) => { data-js="permissions" data-cy="permissions" > - Add Permission + Add Permissions