diff --git a/pom.xml b/pom.xml index e8a533462..d40e7036f 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ waggle-dance-boot waggle-dance-integration-tests waggle-dance - waggle-dance-rpm + @@ -43,7 +43,7 @@ 3.1.5 1.9 1.8.9 - 4.0.0 + 5.0.0 23.0 4.0 1.2.3 @@ -81,6 +81,38 @@ javax.el 2.2.6 + + junit + junit + ${junit.version} + test + + + org.hamcrest + hamcrest-core + + + + + org.mockito + mockito-core + ${mockito.version} + test + + + + net.bytebuddy + byte-buddy + 1.10.15 + test + + + net.bytebuddy + byte-buddy-agent + 1.10.15 + test + org.springframework.boot spring-boot-dependencies @@ -118,6 +150,12 @@ ${hive.version} compile + + org.hamcrest + hamcrest + ${hamcrest.version} + test + org.yaml snakeyaml @@ -126,7 +164,8 @@ org.apache.derby derby - + 10.10.2.0 test @@ -135,6 +174,12 @@ beeju ${beeju.version} test + + + org.eclipse.jetty + jetty-runner + + com.hotels @@ -153,6 +198,8 @@ ${maven.release.plugin.version} + org.codehaus.mojo aspectj-maven-plugin ${aspectj-maven-plugin.version} @@ -224,15 +271,7 @@ - src/main/java/com/hotels/bdp/waggledance/api/validation/constraint/TunnelRoute.java - src/main/java/com/hotels/bdp/waggledance/api/validation/validator/TunnelRouteValidator.java - - src/main/java/com/hotels/bdp/waggledance/client/ThriftMetastoreClient.java src/main/java/com/hotels/bdp/waggledance/server/MetaStoreProxyServer.java - src/main/java/com/hotels/bdp/waggledance/validation/Preconditions.java - src/main/java/com/hotels/bdp/waggledance/parse/ASTConverter.java - src/main/java/com/hotels/bdp/waggledance/parse/ASTNodeUtils.java - src/main/java/com/hotels/bdp/waggledance/parse/Rule.java diff --git a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java index dc53571c5..7f7ff2bc9 100644 --- a/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java +++ b/waggle-dance-api/src/main/java/com/hotels/bdp/waggledance/api/model/AbstractMetaStore.java @@ -43,6 +43,7 @@ @Type(value = PrimaryMetaStore.class, name = "PRIMARY"), @Type(value = FederatedMetaStore.class, name = "FEDERATED") }) public abstract class AbstractMetaStore { + private String catalog = "hive"; private String databasePrefix; private String hiveMetastoreFilterHook; private List writableDatabaseWhitelist; @@ -91,6 +92,16 @@ public static PrimaryMetaStore newPrimaryInstance(String name, String remoteMeta return new PrimaryMetaStore(name, remoteMetaStoreUris, AccessControlType.READ_ONLY); } + public String getCatalog() + { + return catalog; + } + + public void setCatalog(String catalog) + { + this.catalog = catalog; + } + public String getDatabasePrefix() { return databasePrefix; } @@ -212,7 +223,7 @@ public void setStatus(MetaStoreStatus status) { @Override public int hashCode() { - return Objects.hashCode(name); + return Objects.hashCode(name, catalog); } @Override @@ -224,7 +235,7 @@ public boolean equals(Object obj) { return false; } final AbstractMetaStore other = (AbstractMetaStore) obj; - return Objects.equal(name, other.name); + return Objects.equal(name, other.name) && Objects.equal(catalog, other.catalog); } @Override diff --git a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java index 86048bb53..666a0abf1 100644 --- a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java +++ b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/FederatedMetaStoreTest.java @@ -72,7 +72,7 @@ public void nullDatabasePrefix() { @Test public void toJson() throws Exception { - String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; + String expected = "{\"accessControlType\":\"READ_ONLY\",\"catalog\":\"hive\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"name_\",\"federationType\":\"FEDERATED\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; ObjectMapper mapper = new ObjectMapper(); // Sorting to get deterministic test behaviour mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY); diff --git a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java index 8c1634eb6..1887ab078 100644 --- a/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java +++ b/waggle-dance-api/src/test/java/com/hotels/bdp/waggledance/api/model/PrimaryMetaStoreTest.java @@ -89,7 +89,7 @@ public void nonEmptyDatabasePrefix() { @Test public void toJson() throws Exception { - String expected = "{\"accessControlType\":\"READ_ONLY\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; + String expected = "{\"accessControlType\":\"READ_ONLY\",\"catalog\":\"hive\",\"connectionType\":\"DIRECT\",\"databaseNameMapping\":{},\"databasePrefix\":\"\",\"federationType\":\"PRIMARY\",\"hiveMetastoreFilterHook\":null,\"latency\":0,\"mappedDatabases\":null,\"mappedTables\":null,\"metastoreTunnel\":null,\"name\":\"name\",\"remoteMetaStoreUris\":\"uri\",\"status\":\"UNKNOWN\",\"writableDatabaseWhiteList\":[]}"; ObjectMapper mapper = new ObjectMapper(); // Sorting to get deterministic test behaviour mapper.enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY); diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMapping.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMapping.java index 283f84ecc..640e0146a 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMapping.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMapping.java @@ -55,7 +55,6 @@ public interface DatabaseMapping extends MetaStoreMapping { - Partition transformInboundPartition(Partition partition); Table transformInboundTable(Table table); diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMappingImpl.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMappingImpl.java index bafdf6777..ae5030521 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMappingImpl.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMappingImpl.java @@ -80,6 +80,12 @@ public DatabaseMappingImpl(MetaStoreMapping metaStoreMapping, QueryMapping query this.queryMapping = queryMapping; } + @Override + public String getCatalog() + { + return metaStoreMapping.getCatalog(); + } + @Override public MetaStoreFilterHook getMetastoreFilter() { return metaStoreMapping.getMetastoreFilter(); diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMapping.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMapping.java index c8ecf2d00..02f870dc7 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMapping.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMapping.java @@ -30,6 +30,8 @@ public interface MetaStoreMapping extends Closeable { + String getCatalog(); + /** * Outbound means parameter coming from the Hive Metastore and return result will be sent to user client. * diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecorator.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecorator.java index ad795f0f3..ba3f10bde 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecorator.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecorator.java @@ -35,6 +35,12 @@ public MetaStoreMappingDecorator(MetaStoreMapping metaStoreMapping) { this.metaStoreMapping = metaStoreMapping; } + @Override + public String getCatalog() + { + return metaStoreMapping.getCatalog(); + } + @Override public String transformOutboundDatabaseName(String databaseName) { if (databaseName == null) { diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java index c4efbc590..0f696a07e 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingFactoryImpl.java @@ -76,7 +76,7 @@ public MetaStoreMapping newInstance(AbstractMetaStore metaStore) { LOG .info("Mapping databases with name '{}' to metastore: {}", metaStore.getName(), metaStore.getRemoteMetaStoreUris()); - MetaStoreMapping metaStoreMapping = new MetaStoreMappingImpl(prefixNameFor(metaStore), metaStore.getName(), + MetaStoreMapping metaStoreMapping = new MetaStoreMappingImpl(prefixNameFor(metaStore), metaStore.getName(),metaStore.getCatalog(), createClient(metaStore), accessControlHandlerFactory.newInstance(metaStore), metaStore.getConnectionType(), metaStore.getLatency(), loadMetastoreFilterHook(metaStore)); if (waggleDanceConfiguration.getDatabaseResolution() == DatabaseResolution.PREFIXED) { diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java index fc9711f5f..bcd4b18a8 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImpl.java @@ -15,6 +15,8 @@ */ package com.hotels.bdp.waggledance.mapping.model; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.*; + import java.io.IOException; import java.util.Collections; import java.util.List; @@ -32,6 +34,7 @@ import com.hotels.bdp.waggledance.api.model.ConnectionType; import com.hotels.bdp.waggledance.client.CloseableThriftHiveMetastoreIface; +import com.hotels.bdp.waggledance.server.FederatedHMSHandler; import com.hotels.bdp.waggledance.server.security.AccessControlHandler; import com.hotels.bdp.waggledance.server.security.NotAllowedException; @@ -43,6 +46,7 @@ class MetaStoreMappingImpl implements MetaStoreMapping { private final CloseableThriftHiveMetastoreIface client; private final AccessControlHandler accessControlHandler; private final String name; + private final String catalog; private final long latency; private final MetaStoreFilterHook metastoreFilter; @@ -51,6 +55,7 @@ class MetaStoreMappingImpl implements MetaStoreMapping { MetaStoreMappingImpl( String databasePrefix, String name, + String catalog, CloseableThriftHiveMetastoreIface client, AccessControlHandler accessControlHandler, ConnectionType connectionType, @@ -58,6 +63,7 @@ class MetaStoreMappingImpl implements MetaStoreMapping { MetaStoreFilterHook metastoreFilter) { this.databasePrefix = databasePrefix; this.name = name; + this.catalog = catalog; this.client = client; this.accessControlHandler = accessControlHandler; this.connectionType = connectionType; @@ -65,6 +71,11 @@ class MetaStoreMappingImpl implements MetaStoreMapping { this.metastoreFilter = metastoreFilter; } + public String getCatalog() + { + return catalog; + } + @Override public String transformOutboundDatabaseName(String databaseName) { return databaseName.toLowerCase(Locale.ROOT); @@ -121,10 +132,11 @@ public boolean isAvailable() { } @Override - public MetaStoreMapping checkWritePermissions(String databaseName) { - if (!accessControlHandler.hasWritePermission(databaseName)) { + public MetaStoreMapping checkWritePermissions(String databaseName){ + String internal_name = FederatedHMSHandler.getDbInternalName(databaseName); + if (!accessControlHandler.hasWritePermission(internal_name)) { throw new NotAllowedException( - "You cannot perform this operation on the virtual database '" + databaseName + "'."); + "You cannot perform this operation on the virtual database '" + databaseName + "'."); } return this; } diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/PrefixMapping.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/PrefixMapping.java index 5b7b8f9f8..cadf1d9d4 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/PrefixMapping.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/model/PrefixMapping.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2020 Expedia, Inc. + * Copyright (C) 2016-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,16 @@ */ package com.hotels.bdp.waggledance.mapping.model; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.DB_NAME; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependNotNullCatToDbName; + import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.hive.metastore.api.MetaException; + public class PrefixMapping extends MetaStoreMappingDecorator { public PrefixMapping(MetaStoreMapping metaStoreMapping) { @@ -25,8 +32,28 @@ public PrefixMapping(MetaStoreMapping metaStoreMapping) { } @Override - public String transformOutboundDatabaseName(String databaseName) { - return getDatabasePrefix() + super.transformOutboundDatabaseName(databaseName); + public String transformOutboundDatabaseName(String databaseName) { + if(hasCatalogName(databaseName)) { + String[] catalogAndDatabase = new String[0]; + try { + catalogAndDatabase = parseDbName(databaseName, null); + } + catch (MetaException e) { + //FIXME + throw new RuntimeException(e); + } + String dbName = catalogAndDatabase[DB_NAME]; + String catalogName = catalogAndDatabase[CAT_NAME]; + String prefixedDbName = getDatabasePrefix() + super.transformOutboundDatabaseName(dbName); + return prependNotNullCatToDbName(catalogName, prefixedDbName); + }else { + return getDatabasePrefix() + super.transformOutboundDatabaseName(databaseName); + } + } + + private static boolean hasCatalogName(String dbName) { + return dbName != null && dbName.length() > 0 && + dbName.charAt(0) == '@'; } @Override @@ -34,13 +61,30 @@ public List transformOutboundDatabaseNameMultiple(String databaseName) { List outbound = super.transformOutboundDatabaseNameMultiple(databaseName); List result = new ArrayList<>(outbound.size()); for (String outboundDatabase : outbound) { - result.add(getDatabasePrefix() + outboundDatabase); + result.add(transformOutboundDatabaseName(outboundDatabase)); } return result; } @Override public String transformInboundDatabaseName(String databaseName) { + if(hasCatalogName(databaseName)) { + String[] catalogAndDatabase = new String[0]; + try { + catalogAndDatabase = parseDbName(databaseName, null); + } + catch (MetaException e) { + //FIXME + throw new RuntimeException(e); + } + String dbName = catalogAndDatabase[DB_NAME]; + String catalogName = catalogAndDatabase[CAT_NAME]; + return prependNotNullCatToDbName(catalogName, internalTransformInboundDatabaseName(dbName)); + } + return internalTransformInboundDatabaseName(databaseName); + } + + private String internalTransformInboundDatabaseName(String databaseName) { String result = super.transformInboundDatabaseName(databaseName); if (result.startsWith(getDatabasePrefix())) { return result.substring(getDatabasePrefix().length()); diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticOperationHandler.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticOperationHandler.java index 26d041a33..73e4d9ac4 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticOperationHandler.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/PanopticOperationHandler.java @@ -26,7 +26,9 @@ import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.api.GetAllFunctionsResponse; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.TableMeta; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import com.hotels.bdp.waggledance.mapping.model.DatabaseMapping; import com.hotels.bdp.waggledance.mapping.service.requests.GetAllDatabasesByPatternRequest; @@ -59,7 +61,8 @@ public abstract class PanopticOperationHandler { * @param databasePattern pattern to match * @return list of all databases that match the passed pattern */ - public abstract List getAllDatabases(String databasePattern); + public abstract List getAllDatabases(String databasePattern) + throws MetaException; protected List getAllDatabases( Map databaseMappingsForPattern, @@ -69,7 +72,7 @@ protected List getAllDatabases( for (Entry mappingWithPattern : databaseMappingsForPattern.entrySet()) { DatabaseMapping mapping = mappingWithPattern.getKey(); GetAllDatabasesByPatternRequest databasesByPatternRequest = new GetAllDatabasesByPatternRequest(mapping, - mappingWithPattern.getValue(), filter); + createInboundPattern(mappingWithPattern, mapping), filter); allRequests.add(databasesByPatternRequest); } List result = getPanopticOperationExecutor() @@ -77,6 +80,16 @@ protected List getAllDatabases( return result; } + private String createInboundPattern(Entry mappingWithPattern, DatabaseMapping mapping) + { + String pattern = mappingWithPattern.getValue(); + if(pattern.startsWith("@")) { + String sanifiedPattern = mappingWithPattern.getValue().equals("*") ? null : mappingWithPattern.getValue(); + pattern = MetaStoreUtils.prependCatalogToDbName(mapping.getCatalog(), pattern,null); + } + return pattern; + } + /** * Implements {@link HMSHandler#get_table_meta(String, String, List)} over multiple metastores * @@ -85,7 +98,8 @@ protected List getAllDatabases( * @param tableTypes table types to match * @return list of table metadata */ - abstract public List getTableMeta(String databasePatterns, String tablePatterns, List tableTypes); + abstract public List getTableMeta(String databasePatterns, String tablePatterns, List tableTypes) + throws MetaException; protected List getTableMeta( String tablePatterns, @@ -96,7 +110,7 @@ protected List getTableMeta( for (Entry mappingWithPattern : databaseMappingsForPattern.entrySet()) { DatabaseMapping mapping = mappingWithPattern.getKey(); - GetTableMetaRequest tableMetaRequest = new GetTableMetaRequest(mapping, mappingWithPattern.getValue(), + GetTableMetaRequest tableMetaRequest = new GetTableMetaRequest(mapping, createInboundPattern(mappingWithPattern, mapping), tablePatterns, tableTypes, filter); allRequests.add(tableMetaRequest); } diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/impl/PrefixBasedDatabaseMappingService.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/impl/PrefixBasedDatabaseMappingService.java index 8c733df91..c1f59b8cf 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/impl/PrefixBasedDatabaseMappingService.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/impl/PrefixBasedDatabaseMappingService.java @@ -57,6 +57,7 @@ import com.hotels.bdp.waggledance.mapping.service.PanopticOperationExecutor; import com.hotels.bdp.waggledance.mapping.service.PanopticOperationHandler; import com.hotels.bdp.waggledance.mapping.service.requests.GetAllDatabasesRequest; +import com.hotels.bdp.waggledance.server.FederatedHMSHandler; import com.hotels.bdp.waggledance.server.NoPrimaryMetastoreException; import com.hotels.bdp.waggledance.util.AllowList; @@ -294,15 +295,16 @@ private List getMappedAllowedDatabases(List databases, DatabaseM } private boolean isDbAllowed(String databasePrefix, String database) { + String internal_name = FederatedHMSHandler.getDbInternalName(database); AllowList allowList = mappedDbByPrefix.get(databasePrefix); if (allowList == null) { // Accept everything return true; } - return allowList.contains(database); + return allowList.contains(internal_name); } - private boolean databaseAndTableAllowed(String database, String table, DatabaseMapping mapping) { + private boolean databaseAndTableAllowed(String database, String table, DatabaseMapping mapping) { String dbPrefix = mapping.getDatabasePrefix(); boolean databaseAllowed = isDbAllowed(dbPrefix, database); boolean tableAllowed = isTableAllowed(dbPrefix, database, table); @@ -315,7 +317,8 @@ public PanopticOperationHandler getPanopticOperationHandler() { @Override public List getTableMeta(String db_patterns, String tbl_patterns, List tbl_types) { - Map databaseMappingsForPattern = databaseMappingsByDbPattern(db_patterns); + String internal_pattern = FederatedHMSHandler.getDbInternalName(db_patterns); + Map databaseMappingsForPattern = databaseMappingsByDbPattern(internal_pattern); BiFunction filter = (tableMeta, mapping) -> databaseAndTableAllowed( tableMeta.getDbName(), tableMeta.getTableName(), mapping); @@ -325,10 +328,11 @@ public List getTableMeta(String db_patterns, String tbl_patterns, Lis @Override public List getAllDatabases(String databasePattern) { - Map databaseMappingsForPattern = databaseMappingsByDbPattern(databasePattern); + String internal_pattern = FederatedHMSHandler.getDbInternalName(databasePattern); + Map databaseMappingsForPattern = databaseMappingsByDbPattern(internal_pattern); BiFunction filter = (database, mapping) -> isDbAllowed( - mapping.getDatabasePrefix(), database); + mapping.getDatabasePrefix(), database); return super.getAllDatabases(databaseMappingsForPattern, filter); } diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/impl/StaticDatabaseMappingService.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/impl/StaticDatabaseMappingService.java index 5f52b89ab..cdd70d563 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/impl/StaticDatabaseMappingService.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/mapping/service/impl/StaticDatabaseMappingService.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2020 Expedia, Inc. + * Copyright (C) 2016-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -63,6 +63,7 @@ import com.hotels.bdp.waggledance.mapping.service.PanopticConcurrentOperationExecutor; import com.hotels.bdp.waggledance.mapping.service.PanopticOperationExecutor; import com.hotels.bdp.waggledance.mapping.service.PanopticOperationHandler; +import com.hotels.bdp.waggledance.server.FederatedHMSHandler; import com.hotels.bdp.waggledance.server.NoPrimaryMetastoreException; import com.hotels.bdp.waggledance.util.AllowList; @@ -292,7 +293,7 @@ private boolean includeInResults(MetaStoreMapping metaStoreMapping) { @Override public DatabaseMapping databaseMapping(@NotNull String databaseName) throws NoSuchObjectException { - DatabaseMapping databaseMapping = mappingsByDatabaseName.get(databaseName.toLowerCase(Locale.ROOT)); + DatabaseMapping databaseMapping = mappingsByDatabaseName.get(FederatedHMSHandler.getDbInternalName(databaseName.toLowerCase(Locale.ROOT))); if (databaseMapping != null) { LOG .debug("Database Name `{}` maps to metastore with name '{}'", databaseName, @@ -360,28 +361,29 @@ public PanopticOperationHandler getPanopticOperationHandler() { @Override public List getTableMeta(String db_patterns, String tbl_patterns, List tbl_types) { + String internal_pattern = FederatedHMSHandler.getDbInternalName(db_patterns); BiFunction filter = (tableMeta, mapping) -> databaseAndTableAllowed(tableMeta.getDbName(), tableMeta.getTableName(), mapping); Map mappingsForPattern = new LinkedHashMap<>(); for (DatabaseMapping mapping : getDatabaseMappings()) { - mappingsForPattern.put(mapping, db_patterns); + mappingsForPattern.put(mapping, internal_pattern); } return super.getTableMeta(tbl_patterns, tbl_types, mappingsForPattern, filter); } @Override public List getAllDatabases(String pattern) { - BiFunction filter = (database, mapping) -> mappingsByDatabaseName - .containsKey(database); + String internal_pattern = FederatedHMSHandler.getDbInternalName(pattern); + BiFunction filter = (database, mapping) -> isDbAllowed(database); Map mappingsForPattern = new LinkedHashMap<>(); for (DatabaseMapping mapping : getDatabaseMappings()) { - mappingsForPattern.put(mapping, pattern); + mappingsForPattern.put(mapping, internal_pattern); } - return super.getAllDatabases(mappingsForPattern, filter); + return new ArrayList(new HashSet(super.getAllDatabases(mappingsForPattern, filter))); } @Override @@ -396,6 +398,12 @@ protected PanopticOperationExecutor getPanopticOperationExecutor() { }; } + private boolean isDbAllowed(String database) + { + String internal_name = FederatedHMSHandler.getDbInternalName(database); + return mappingsByDatabaseName.containsKey(internal_name); + } + @Override public void close() throws IOException { if (mappingsByMetaStoreName != null) { diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java index 0cd61e3ef..a867641f8 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandler.java @@ -15,6 +15,8 @@ */ package com.hotels.bdp.waggledance.server; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.*; + import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -109,8 +111,12 @@ import org.apache.hadoop.hive.metastore.api.PutFileMetadataRequest; import org.apache.hadoop.hive.metastore.api.PutFileMetadataResult; import org.apache.hadoop.hive.metastore.api.Role; +import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; +import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; +import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; import org.apache.hadoop.hive.metastore.api.ShowCompactRequest; import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; @@ -147,14 +153,14 @@ @Monitored @Component @Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) -class FederatedHMSHandler extends FacebookBase implements CloseableIHMSHandler { +public abstract class FederatedHMSHandler extends FacebookBase implements CloseableIHMSHandler { private static final Logger LOG = LoggerFactory.getLogger(FederatedHMSHandler.class); - private static final String INVOCATION_LOG_NAME = "com.hotels.bdp.waggledance.server.invocation-log"; - private final MappingEventListener databaseMappingService; - private final NotifyingFederationService notifyingFederationService; - private Configuration conf; + static final String INVOCATION_LOG_NAME = "com.hotels.bdp.waggledance.server.invocation-log"; + MappingEventListener databaseMappingService; + NotifyingFederationService notifyingFederationService; + Configuration conf; FederatedHMSHandler( MappingEventListener databaseMappingService, @@ -165,7 +171,7 @@ class FederatedHMSHandler extends FacebookBase implements CloseableIHMSHandler { this.notifyingFederationService.subscribe(databaseMappingService); } - private ThriftHiveMetastore.Iface getPrimaryClient() throws TException { + ThriftHiveMetastore.Iface getPrimaryClient() throws TException { return databaseMappingService.primaryDatabaseMapping().getClient(); } @@ -235,7 +241,8 @@ public void create_database(Database database) @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) public Database get_database(String name) throws NoSuchObjectException, MetaException, TException { LOG.info("Fetching database {}", name); - DatabaseMapping mapping = databaseMappingService.databaseMapping(name); + String internal_name = getDbInternalName(name); + DatabaseMapping mapping = databaseMappingService.databaseMapping(internal_name); LOG.info("Mapping is '{}'", mapping.getDatabasePrefix()); Database result = mapping.getClient().get_database(mapping.transformInboundDatabaseName(name)); return mapping.transformOutboundDatabase(mapping.getMetastoreFilter().filterDatabase(result)); @@ -264,8 +271,11 @@ public List get_all_databases() throws MetaException, TException { @Override @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) public void alter_database(String dbname, Database db) throws MetaException, NoSuchObjectException, TException { - DatabaseMapping mapping = checkWritePermissions(dbname); - mapping.checkWritePermissions(db.getName()); + String internal_name = getDbInternalName(dbname); + DatabaseMapping mapping = checkWritePermissions(internal_name); + // mapping.checkWritePermissions(db.getName()); + // changed for allowing db renaming + mapping.checkWritePermissions(internal_name); mapping .getClient() .alter_database(mapping.transformInboundDatabaseName(dbname), mapping.transformInboundDatabase(db)); @@ -344,7 +354,8 @@ public void drop_table_with_environment_context( boolean deleteData, EnvironmentContext environment_context) throws NoSuchObjectException, MetaException, TException { - DatabaseMapping mapping = checkWritePermissionsAndCheckTableAllowed(dbname, name); + String internal_name = getDbInternalName(dbname); + DatabaseMapping mapping = checkWritePermissionsAndCheckTableAllowed(internal_name, name); mapping .getClient() .drop_table_with_environment_context(mapping.transformInboundDatabaseName(dbname), name, deleteData, @@ -357,24 +368,24 @@ public List get_tables(String db_name, String pattern) throws MetaExcept DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name); List resultTables = mapping.getClient().get_tables(mapping.transformInboundDatabaseName(db_name), pattern); resultTables = databaseMappingService.filterTables(db_name, resultTables, mapping); - return mapping.getMetastoreFilter().filterTableNames(db_name, resultTables); + return mapping.getMetastoreFilter().filterTableNames(mapping.getCatalog(),db_name, resultTables); } - - @Override @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) public List get_all_tables(String db_name) throws MetaException, TException { - DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name); + String internal_name = getDbInternalName(db_name); + DatabaseMapping mapping = databaseMappingService.databaseMapping(internal_name); List resultTables = mapping.getClient().get_all_tables(mapping.transformInboundDatabaseName(db_name)); - resultTables = databaseMappingService.filterTables(db_name, resultTables, mapping); - return mapping.getMetastoreFilter().filterTableNames(db_name, resultTables); + resultTables = databaseMappingService.filterTables(internal_name, resultTables, mapping); + return mapping.getMetastoreFilter().filterTableNames(mapping.getCatalog(), internal_name, resultTables); } @Override @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) public Table get_table(String dbname, String tbl_name) throws MetaException, NoSuchObjectException, TException { - DatabaseMapping mapping = getDbMappingAndCheckTableAllowed(dbname, tbl_name); + String internal_name = getDbInternalName(dbname); + DatabaseMapping mapping = getDbMappingAndCheckTableAllowed(internal_name, tbl_name); Table table = mapping.getClient().get_table(mapping.transformInboundDatabaseName(dbname), tbl_name); return mapping .transformOutboundTable(mapping.getMetastoreFilter().filterTable(table)); @@ -405,7 +416,7 @@ public List get_table_names_by_filter(String dbname, String filter, shor List resultTables = mapping.getClient() .get_table_names_by_filter(mapping.transformInboundDatabaseName(dbname), filter, max_tables); List result = databaseMappingService.filterTables(dbname, resultTables, mapping); - return mapping.getMetastoreFilter().filterTableNames(dbname, result); + return mapping.getMetastoreFilter().filterTableNames(mapping.getCatalog(), dbname, result); } @Override @@ -722,7 +733,7 @@ public List get_partition_names(String db_name, String tbl_name, short m DatabaseMapping mapping = getDbMappingAndCheckTableAllowed(db_name, tbl_name); List result = mapping.getClient() .get_partition_names(mapping.transformInboundDatabaseName(db_name), tbl_name, max_parts); - return mapping.getMetastoreFilter().filterPartitionNames(db_name, tbl_name, result); + return mapping.getMetastoreFilter().filterPartitionNames(mapping.getCatalog(), db_name, tbl_name, result); } @Override @@ -762,7 +773,7 @@ public List get_partition_names_ps(String db_name, String tbl_name, List List result = mapping .getClient() .get_partition_names_ps(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals, max_parts); - return mapping.getMetastoreFilter().filterPartitionNames(db_name, tbl_name, result); + return mapping.getMetastoreFilter().filterPartitionNames(mapping.getCatalog(), db_name, tbl_name, result); } @Override @@ -803,10 +814,11 @@ public PartitionsByExprResult get_partitions_by_expr(PartitionsByExprRequest req @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME, prepend=true) public List get_partitions_by_names(String db_name, String tbl_name, List names) throws MetaException, NoSuchObjectException, TException { - DatabaseMapping mapping = getDbMappingAndCheckTableAllowed(db_name, tbl_name); + String internal_name = getDbInternalName(db_name); + DatabaseMapping mapping = getDbMappingAndCheckTableAllowed(internal_name, tbl_name); List partitions = mapping .getClient() - .get_partitions_by_names(mapping.transformInboundDatabaseName(db_name), tbl_name, names); + .get_partitions_by_names(mapping.transformInboundDatabaseName(internal_name), tbl_name, names); return mapping.transformOutboundPartitions(mapping.getMetastoreFilter().filterPartitions(partitions)); } @@ -1426,10 +1438,11 @@ public ClearFileMetadataResult clear_file_metadata(ClearFileMetadataRequest req) @Override @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) - public void create_table_with_constraints(Table tbl, List primaryKeys, List foreignKeys) - throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException { + public void create_table_with_constraints(Table tbl, List primaryKeys, List foreignKeys, List uniqueConstraints, List notNullConstraints, List defaultConstraints, List checkConstraints) + throws TException + { DatabaseMapping mapping = checkWritePermissions(tbl.getDbName()); - mapping.getClient().create_table_with_constraints(mapping.transformInboundTable(tbl), primaryKeys, foreignKeys); + mapping.getClient().create_table_with_constraints(mapping.transformInboundTable(tbl), primaryKeys, foreignKeys,uniqueConstraints,notNullConstraints,defaultConstraints,checkConstraints); } @Override @@ -1618,18 +1631,30 @@ public List get_tables_by_type(String db_name, String pattern, String ta DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name); List resultTables = mapping.getClient().get_tables_by_type(mapping.transformInboundDatabaseName(db_name), pattern, tableType); List result = databaseMappingService.filterTables(db_name, resultTables, mapping); - return mapping.getMetastoreFilter().filterTableNames(db_name, result); + return mapping.getMetastoreFilter().filterTableNames(mapping.getCatalog(), db_name, result); } @Override @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) public GetTableResult get_table_req(GetTableRequest req) throws MetaException, NoSuchObjectException, TException { - DatabaseMapping mapping = getDbMappingAndCheckTableAllowed(req.getDbName(), req.getTblName()); + String internal_name = getDbInternalName(req.getDbName()); + DatabaseMapping mapping = getDbMappingAndCheckTableAllowed(internal_name, req.getTblName()); GetTableResult result = mapping.getClient().get_table_req(mapping.transformInboundGetTableRequest(req)); result.setTable(mapping.getMetastoreFilter().filterTable(result.getTable())); return mapping.transformOutboundGetTableResult(result); } + public static String getDbInternalName(String dbName) + { + try { + String internalName = parseDbName(dbName, null)[DB_NAME]; + return internalName!=null ? internalName : "*"; + } + catch (MetaException e) { + throw new RuntimeException(e); + } + } + @Override @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) public GetTablesResult get_table_objects_by_name_req(GetTablesRequest req) diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerFactory.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerFactory.java index 653959219..0c892691b 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerFactory.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerFactory.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016-2020 Expedia, Inc. + * Copyright (C) 2016-2021 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -57,7 +57,7 @@ public CloseableIHMSHandler create() { MappingEventListener service = createDatabaseMappingService(); MonitoredDatabaseMappingService monitoredService = new MonitoredDatabaseMappingService(service); - CloseableIHMSHandler baseHandler = new FederatedHMSHandler(monitoredService, notifyingFederationService); + CloseableIHMSHandler baseHandler = new FederatedHMSHandlerHive3(monitoredService, notifyingFederationService); HiveConf conf = new HiveConf(hiveConf); baseHandler.setConf(conf); return baseHandler; diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerHive3.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerHive3.java new file mode 100644 index 000000000..240bba69a --- /dev/null +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerHive3.java @@ -0,0 +1,674 @@ +/** + * Copyright (C) 2016-2021 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.hotels.bdp.waggledance.server; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hadoop.hive.metastore.MetaStoreEventListener; +import org.apache.hadoop.hive.metastore.RawStore; +import org.apache.hadoop.hive.metastore.TransactionalMetaStoreEventListener; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.AddCheckConstraintRequest; +import org.apache.hadoop.hive.metastore.api.AddDefaultConstraintRequest; +import org.apache.hadoop.hive.metastore.api.AddNotNullConstraintRequest; +import org.apache.hadoop.hive.metastore.api.AddUniqueConstraintRequest; +import org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsRequest; +import org.apache.hadoop.hive.metastore.api.AllocateTableWriteIdsResponse; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.AlterCatalogRequest; +import org.apache.hadoop.hive.metastore.api.AlterISchemaRequest; +import org.apache.hadoop.hive.metastore.api.CheckConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.CheckConstraintsResponse; +import org.apache.hadoop.hive.metastore.api.CmRecycleRequest; +import org.apache.hadoop.hive.metastore.api.CmRecycleResponse; +import org.apache.hadoop.hive.metastore.api.CreateCatalogRequest; +import org.apache.hadoop.hive.metastore.api.CreationMetadata; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.DefaultConstraintsResponse; +import org.apache.hadoop.hive.metastore.api.DropCatalogRequest; +import org.apache.hadoop.hive.metastore.api.FindSchemasByColsResp; +import org.apache.hadoop.hive.metastore.api.FindSchemasByColsRqst; +import org.apache.hadoop.hive.metastore.api.GetCatalogRequest; +import org.apache.hadoop.hive.metastore.api.GetCatalogResponse; +import org.apache.hadoop.hive.metastore.api.GetCatalogsResponse; +import org.apache.hadoop.hive.metastore.api.GetRuntimeStatsRequest; +import org.apache.hadoop.hive.metastore.api.GetSerdeRequest; +import org.apache.hadoop.hive.metastore.api.GetValidWriteIdsRequest; +import org.apache.hadoop.hive.metastore.api.GetValidWriteIdsResponse; +import org.apache.hadoop.hive.metastore.api.GrantRevokePrivilegeRequest; +import org.apache.hadoop.hive.metastore.api.GrantRevokePrivilegeResponse; +import org.apache.hadoop.hive.metastore.api.HiveObjectRef; +import org.apache.hadoop.hive.metastore.api.ISchema; +import org.apache.hadoop.hive.metastore.api.ISchemaName; +import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.LockResponse; +import org.apache.hadoop.hive.metastore.api.MapSchemaVersionToSerdeRequest; +import org.apache.hadoop.hive.metastore.api.Materialization; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.NoSuchTxnException; +import org.apache.hadoop.hive.metastore.api.NotNullConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.NotNullConstraintsResponse; +import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest; +import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse; +import org.apache.hadoop.hive.metastore.api.ReplTblWriteIdStateRequest; +import org.apache.hadoop.hive.metastore.api.RuntimeStat; +import org.apache.hadoop.hive.metastore.api.SchemaVersion; +import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; +import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.api.SetSchemaVersionStateRequest; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.TxnAbortedException; +import org.apache.hadoop.hive.metastore.api.UniqueConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.UniqueConstraintsResponse; +import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.hadoop.hive.metastore.api.WMAlterPoolRequest; +import org.apache.hadoop.hive.metastore.api.WMAlterPoolResponse; +import org.apache.hadoop.hive.metastore.api.WMAlterResourcePlanRequest; +import org.apache.hadoop.hive.metastore.api.WMAlterResourcePlanResponse; +import org.apache.hadoop.hive.metastore.api.WMAlterTriggerRequest; +import org.apache.hadoop.hive.metastore.api.WMAlterTriggerResponse; +import org.apache.hadoop.hive.metastore.api.WMCreateOrDropTriggerToPoolMappingRequest; +import org.apache.hadoop.hive.metastore.api.WMCreateOrDropTriggerToPoolMappingResponse; +import org.apache.hadoop.hive.metastore.api.WMCreateOrUpdateMappingRequest; +import org.apache.hadoop.hive.metastore.api.WMCreateOrUpdateMappingResponse; +import org.apache.hadoop.hive.metastore.api.WMCreatePoolRequest; +import org.apache.hadoop.hive.metastore.api.WMCreatePoolResponse; +import org.apache.hadoop.hive.metastore.api.WMCreateResourcePlanRequest; +import org.apache.hadoop.hive.metastore.api.WMCreateResourcePlanResponse; +import org.apache.hadoop.hive.metastore.api.WMCreateTriggerRequest; +import org.apache.hadoop.hive.metastore.api.WMCreateTriggerResponse; +import org.apache.hadoop.hive.metastore.api.WMDropMappingRequest; +import org.apache.hadoop.hive.metastore.api.WMDropMappingResponse; +import org.apache.hadoop.hive.metastore.api.WMDropPoolRequest; +import org.apache.hadoop.hive.metastore.api.WMDropPoolResponse; +import org.apache.hadoop.hive.metastore.api.WMDropResourcePlanRequest; +import org.apache.hadoop.hive.metastore.api.WMDropResourcePlanResponse; +import org.apache.hadoop.hive.metastore.api.WMDropTriggerRequest; +import org.apache.hadoop.hive.metastore.api.WMDropTriggerResponse; +import org.apache.hadoop.hive.metastore.api.WMGetActiveResourcePlanRequest; +import org.apache.hadoop.hive.metastore.api.WMGetActiveResourcePlanResponse; +import org.apache.hadoop.hive.metastore.api.WMGetAllResourcePlanRequest; +import org.apache.hadoop.hive.metastore.api.WMGetAllResourcePlanResponse; +import org.apache.hadoop.hive.metastore.api.WMGetResourcePlanRequest; +import org.apache.hadoop.hive.metastore.api.WMGetResourcePlanResponse; +import org.apache.hadoop.hive.metastore.api.WMGetTriggersForResourePlanRequest; +import org.apache.hadoop.hive.metastore.api.WMGetTriggersForResourePlanResponse; +import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanRequest; +import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; +import org.apache.hadoop.hive.metastore.txn.TxnStore; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; + +import com.jcabi.aspects.Loggable; + +import com.hotels.bdp.waggledance.mapping.service.MappingEventListener; +import com.hotels.bdp.waggledance.mapping.service.impl.NotifyingFederationService; +import com.hotels.bdp.waggledance.metrics.Monitored; + +@Monitored +@Component +@Scope(ConfigurableBeanFactory.SCOPE_PROTOTYPE) +public class FederatedHMSHandlerHive3 + extends FederatedHMSHandler +{ + private static final Logger LOG = LoggerFactory.getLogger(FederatedHMSHandlerHive3.class); + + private AtomicInteger id = new AtomicInteger(); + + FederatedHMSHandlerHive3(MappingEventListener databaseMappingService, NotifyingFederationService notifyingFederationService) + { + super(databaseMappingService, notifyingFederationService); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public int getThreadId() + { + return id.incrementAndGet(); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public RawStore getMS() + throws MetaException + { + throw new IllegalStateException("Called internally to metastore only so cannot be called by the proxy"); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public TxnStore getTxnHandler() + { + throw new IllegalStateException("Called internally to metastore only so cannot be called by the proxy"); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public Warehouse getWh() + { + throw new IllegalStateException("Called internally to metastore only so cannot be called by the proxy"); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public Database get_database_core(String catName, String name) + throws NoSuchObjectException, MetaException + { + throw new IllegalStateException("Called internally to metastore only so cannot be called by the proxy"); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public Table get_table_core(String catName, String dbname, String name) + throws MetaException, NoSuchObjectException + { + throw new IllegalStateException("Called internally to metastore only so cannot be called by the proxy"); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public List getTransactionalListeners() + { + throw new IllegalStateException("Called internally to metastore only so cannot be called by the proxy"); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public List getListeners() + { + throw new IllegalStateException("Called internally to metastore only so cannot be called by the proxy"); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void create_catalog(CreateCatalogRequest createCatalogRequest) + throws AlreadyExistsException, InvalidObjectException, MetaException, TException + { + getPrimaryClient().create_catalog(createCatalogRequest); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void alter_catalog(AlterCatalogRequest alterCatalogRequest) + throws NoSuchObjectException, InvalidOperationException, MetaException, TException + { + getPrimaryClient().alter_catalog(alterCatalogRequest); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public GetCatalogResponse get_catalog(GetCatalogRequest getCatalogRequest) + throws NoSuchObjectException, MetaException, TException + { + return getPrimaryClient().get_catalog(getCatalogRequest); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public GetCatalogsResponse get_catalogs() + throws MetaException, TException + { + return getPrimaryClient().get_catalogs(); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void drop_catalog(DropCatalogRequest dropCatalogRequest) + throws NoSuchObjectException, InvalidOperationException, MetaException, TException + { + getPrimaryClient().drop_catalog(dropCatalogRequest); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void add_unique_constraint(AddUniqueConstraintRequest addUniqueConstraintRequest) + throws NoSuchObjectException, MetaException, TException + { + getPrimaryClient().add_unique_constraint(addUniqueConstraintRequest); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void add_not_null_constraint(AddNotNullConstraintRequest addNotNullConstraintRequest) + throws NoSuchObjectException, MetaException, TException + { + getPrimaryClient().add_not_null_constraint(addNotNullConstraintRequest); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void add_default_constraint(AddDefaultConstraintRequest addDefaultConstraintRequest) + throws NoSuchObjectException, MetaException, TException + { + getPrimaryClient().add_default_constraint(addDefaultConstraintRequest); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void add_check_constraint(AddCheckConstraintRequest addCheckConstraintRequest) + throws NoSuchObjectException, MetaException, TException + { + getPrimaryClient().add_check_constraint(addCheckConstraintRequest); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void truncate_table(String dbName, String tableName, List partNames) + throws MetaException, TException + { + getPrimaryClient().truncate_table(dbName, tableName, partNames); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public List get_materialized_views_for_rewriting(String view) + throws MetaException, TException + { + return getPrimaryClient().get_materialized_views_for_rewriting(view); + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public Materialization get_materialization_invalidation_info(CreationMetadata creationMetadata, String s) + throws MetaException, InvalidOperationException, UnknownDBException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void update_creation_metadata(String s, String s1, String s2, CreationMetadata creationMetadata) + throws MetaException, InvalidOperationException, UnknownDBException, TException + { + + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public UniqueConstraintsResponse get_unique_constraints(UniqueConstraintsRequest uniqueConstraintsRequest) + throws MetaException, NoSuchObjectException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public NotNullConstraintsResponse get_not_null_constraints(NotNullConstraintsRequest notNullConstraintsRequest) + throws MetaException, NoSuchObjectException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public DefaultConstraintsResponse get_default_constraints(DefaultConstraintsRequest defaultConstraintsRequest) + throws MetaException, NoSuchObjectException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public CheckConstraintsResponse get_check_constraints(CheckConstraintsRequest checkConstraintsRequest) + throws MetaException, NoSuchObjectException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public GrantRevokePrivilegeResponse refresh_privileges(HiveObjectRef hiveObjectRef, String s, GrantRevokePrivilegeRequest grantRevokePrivilegeRequest) + throws MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void repl_tbl_writeid_state(ReplTblWriteIdStateRequest replTblWriteIdStateRequest) + throws TException + { + + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public GetValidWriteIdsResponse get_valid_write_ids(GetValidWriteIdsRequest getValidWriteIdsRequest) + throws NoSuchTxnException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public AllocateTableWriteIdsResponse allocate_table_write_ids(AllocateTableWriteIdsRequest allocateTableWriteIdsRequest) + throws NoSuchTxnException, TxnAbortedException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public NotificationEventsCountResponse get_notification_events_count(NotificationEventsCountRequest notificationEventsCountRequest) + throws TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public CmRecycleResponse cm_recycle(CmRecycleRequest cmRecycleRequest) + throws MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public String get_metastore_db_uuid() + throws MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMCreateResourcePlanResponse create_resource_plan(WMCreateResourcePlanRequest wmCreateResourcePlanRequest) + throws AlreadyExistsException, InvalidObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMGetResourcePlanResponse get_resource_plan(WMGetResourcePlanRequest wmGetResourcePlanRequest) + throws NoSuchObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMGetActiveResourcePlanResponse get_active_resource_plan(WMGetActiveResourcePlanRequest wmGetActiveResourcePlanRequest) + throws MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMGetAllResourcePlanResponse get_all_resource_plans(WMGetAllResourcePlanRequest wmGetAllResourcePlanRequest) + throws MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMAlterResourcePlanResponse alter_resource_plan(WMAlterResourcePlanRequest wmAlterResourcePlanRequest) + throws NoSuchObjectException, InvalidOperationException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMValidateResourcePlanResponse validate_resource_plan(WMValidateResourcePlanRequest wmValidateResourcePlanRequest) + throws NoSuchObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMDropResourcePlanResponse drop_resource_plan(WMDropResourcePlanRequest wmDropResourcePlanRequest) + throws NoSuchObjectException, InvalidOperationException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMCreateTriggerResponse create_wm_trigger(WMCreateTriggerRequest wmCreateTriggerRequest) + throws AlreadyExistsException, NoSuchObjectException, InvalidObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMAlterTriggerResponse alter_wm_trigger(WMAlterTriggerRequest wmAlterTriggerRequest) + throws NoSuchObjectException, InvalidObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMDropTriggerResponse drop_wm_trigger(WMDropTriggerRequest wmDropTriggerRequest) + throws NoSuchObjectException, InvalidOperationException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMGetTriggersForResourePlanResponse get_triggers_for_resourceplan(WMGetTriggersForResourePlanRequest wmGetTriggersForResourePlanRequest) + throws NoSuchObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMCreatePoolResponse create_wm_pool(WMCreatePoolRequest wmCreatePoolRequest) + throws AlreadyExistsException, NoSuchObjectException, InvalidObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMAlterPoolResponse alter_wm_pool(WMAlterPoolRequest wmAlterPoolRequest) + throws AlreadyExistsException, NoSuchObjectException, InvalidObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMDropPoolResponse drop_wm_pool(WMDropPoolRequest wmDropPoolRequest) + throws NoSuchObjectException, InvalidOperationException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMCreateOrUpdateMappingResponse create_or_update_wm_mapping(WMCreateOrUpdateMappingRequest wmCreateOrUpdateMappingRequest) + throws AlreadyExistsException, NoSuchObjectException, InvalidObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMDropMappingResponse drop_wm_mapping(WMDropMappingRequest wmDropMappingRequest) + throws NoSuchObjectException, InvalidOperationException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public WMCreateOrDropTriggerToPoolMappingResponse create_or_drop_wm_trigger_to_pool_mapping(WMCreateOrDropTriggerToPoolMappingRequest wmCreateOrDropTriggerToPoolMappingRequest) + throws AlreadyExistsException, NoSuchObjectException, InvalidObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void create_ischema(ISchema iSchema) + throws AlreadyExistsException, NoSuchObjectException, MetaException, TException + { + + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void alter_ischema(AlterISchemaRequest alterISchemaRequest) + throws NoSuchObjectException, MetaException, TException + { + + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public ISchema get_ischema(ISchemaName iSchemaName) + throws NoSuchObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void drop_ischema(ISchemaName iSchemaName) + throws NoSuchObjectException, InvalidOperationException, MetaException, TException + { + + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void add_schema_version(SchemaVersion schemaVersion) + throws AlreadyExistsException, NoSuchObjectException, MetaException, TException + { + + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public SchemaVersion get_schema_version(SchemaVersionDescriptor schemaVersionDescriptor) + throws NoSuchObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public SchemaVersion get_schema_latest_version(ISchemaName iSchemaName) + throws NoSuchObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public List get_schema_all_versions(ISchemaName iSchemaName) + throws NoSuchObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void drop_schema_version(SchemaVersionDescriptor schemaVersionDescriptor) + throws NoSuchObjectException, MetaException, TException + { + + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public FindSchemasByColsResp get_schemas_by_cols(FindSchemasByColsRqst findSchemasByColsRqst) + throws MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void map_schema_version_to_serde(MapSchemaVersionToSerdeRequest mapSchemaVersionToSerdeRequest) + throws NoSuchObjectException, MetaException, TException + { + + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void set_schema_version_state(SetSchemaVersionStateRequest setSchemaVersionStateRequest) + throws NoSuchObjectException, InvalidOperationException, MetaException, TException + { + + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void add_serde(SerDeInfo serDeInfo) + throws AlreadyExistsException, MetaException, TException + { + + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public SerDeInfo get_serde(GetSerdeRequest getSerdeRequest) + throws NoSuchObjectException, MetaException, TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public LockResponse get_lock_materialization_rebuild(String s, String s1, long l) + throws TException + { + return null; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public boolean heartbeat_lock_materialization_rebuild(String s, String s1, long l) + throws TException + { + return false; + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public void add_runtime_stats(RuntimeStat runtimeStat) + throws MetaException, TException + { + + } + + @Override + @Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME) + public List get_runtime_stats(GetRuntimeStatsRequest getRuntimeStatsRequest) + throws MetaException, TException + { + return null; + } +} diff --git a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/security/DatabaseWhitelistAccessControlHandler.java b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/security/DatabaseWhitelistAccessControlHandler.java index 9b2a043ac..087ac9b28 100644 --- a/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/security/DatabaseWhitelistAccessControlHandler.java +++ b/waggle-dance-core/src/main/java/com/hotels/bdp/waggledance/server/security/DatabaseWhitelistAccessControlHandler.java @@ -27,63 +27,63 @@ public class DatabaseWhitelistAccessControlHandler implements AccessControlHandler { - private final FederationService federationService; - private final boolean hasCreatePermission; - private final AllowList writableDatabaseWhiteList; - private AbstractMetaStore metaStore; + private final FederationService federationService; + private final boolean hasCreatePermission; + private final AllowList writableDatabaseWhiteList; + private AbstractMetaStore metaStore; - DatabaseWhitelistAccessControlHandler( - AbstractMetaStore metaStore, - FederationService federationService, + DatabaseWhitelistAccessControlHandler( + AbstractMetaStore metaStore, + FederationService federationService, boolean hasCreatePermission) { - this.metaStore = metaStore; - this.federationService = federationService; - this.hasCreatePermission = hasCreatePermission; - writableDatabaseWhiteList = new AllowList(metaStore.getWritableDatabaseWhiteList()); - } + this.metaStore = metaStore; + this.federationService = federationService; + this.hasCreatePermission = hasCreatePermission; + writableDatabaseWhiteList = new AllowList(metaStore.getWritableDatabaseWhiteList()); + } private String trimToLowerCase(String string) { - return string.trim().toLowerCase(Locale.ROOT); - } + return string.trim().toLowerCase(Locale.ROOT); + } - @Override + @Override public boolean hasWritePermission(String databaseName) { return writableDatabaseWhiteList.contains(databaseName); - } + } - @Override + @Override public boolean hasCreatePermission() { - return hasCreatePermission; - } + return hasCreatePermission; + } - @Override + @Override public void databaseCreatedNotification(String name) { - List newWritableDatabaseWhiteList = new ArrayList<>(metaStore.getWritableDatabaseWhiteList()); - List mappedDatabases = null; - String nameLowerCase = trimToLowerCase(name); - if (!newWritableDatabaseWhiteList.contains(nameLowerCase)) { - newWritableDatabaseWhiteList.add(nameLowerCase); - } - if (metaStore.getMappedDatabases() != null) { - mappedDatabases = new ArrayList<>(metaStore.getMappedDatabases()); - if (!mappedDatabases.contains(name)) { - mappedDatabases.add(name); - } - } + List newWritableDatabaseWhiteList = new ArrayList<>(metaStore.getWritableDatabaseWhiteList()); + List mappedDatabases = null; + String nameLowerCase = trimToLowerCase(name); + if (!newWritableDatabaseWhiteList.contains(nameLowerCase)) { + newWritableDatabaseWhiteList.add(nameLowerCase); + } + if (metaStore.getMappedDatabases() != null) { + mappedDatabases = new ArrayList<>(metaStore.getMappedDatabases()); + if (!mappedDatabases.contains(name)) { + mappedDatabases.add(name); + } + } - AbstractMetaStore newMetaStore; - if (metaStore instanceof PrimaryMetaStore) { - newMetaStore = new PrimaryMetaStore(metaStore.getName(), metaStore.getRemoteMetaStoreUris(), - metaStore.getAccessControlType(), newWritableDatabaseWhiteList); - newMetaStore.setMappedDatabases(mappedDatabases); + AbstractMetaStore newMetaStore; + if (metaStore instanceof PrimaryMetaStore) { + newMetaStore = new PrimaryMetaStore(metaStore.getName(), metaStore.getRemoteMetaStoreUris(), + metaStore.getAccessControlType(), newWritableDatabaseWhiteList); + newMetaStore.setMappedDatabases(mappedDatabases); } else { - throw new WaggleDanceException( - String.format("Metastore type %s does not support database creation", metaStore.getClass().getName())); - } + throw new WaggleDanceException( + String.format("Metastore type %s does not support database creation", metaStore.getClass().getName())); + } - federationService.update(metaStore, newMetaStore); - metaStore = newMetaStore; - writableDatabaseWhiteList.add(nameLowerCase); - } + federationService.update(metaStore, newMetaStore); + metaStore = newMetaStore; + writableDatabaseWhiteList.add(nameLowerCase); + } } diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMappingTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMappingTest.java index aa94fd87a..7ce0b5937 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMappingTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/ASTQueryMappingTest.java @@ -38,7 +38,7 @@ public class ASTQueryMappingTest { @Before public void setUp() { - metaStoreMapping = new PrefixMapping(new MetaStoreMappingImpl(PREFIX, "mapping", null, null, DIRECT, LATENCY, + metaStoreMapping = new PrefixMapping(new MetaStoreMappingImpl(PREFIX, "mapping","hive", null, null, DIRECT, LATENCY, new DefaultMetaStoreFilterHookImpl(new HiveConf()))); } diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMappingImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMappingImplTest.java index 1f14e7a9a..31854e3e8 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMappingImplTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseMappingImplTest.java @@ -54,7 +54,6 @@ import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; import org.apache.hadoop.hive.metastore.api.HiveObjectRef; import org.apache.hadoop.hive.metastore.api.HiveObjectType; -import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.LockComponent; import org.apache.hadoop.hive.metastore.api.LockRequest; import org.apache.hadoop.hive.metastore.api.Partition; @@ -99,7 +98,6 @@ public class DatabaseMappingImplTest { private DatabaseMappingImpl databaseMapping; private Partition partition; - private Index index; private HiveObjectRef hiveObjectRef; private PartitionSpec partitionSpec; private Database database; @@ -114,8 +112,6 @@ public void setUp() { partition = new Partition(); partition.setDbName(DB_NAME); partitions = Lists.newArrayList(partition); - index = new Index(); - index.setDbName(DB_NAME); hiveObjectRef = new HiveObjectRef(); hiveObjectRef.setDbName(DB_NAME); hiveObjectRef.setObjectType(HiveObjectType.DATABASE); @@ -241,20 +237,6 @@ public void transformInboundPartition() throws Exception { assertThat(result.getDbName(), is(IN_DB_NAME)); } - @Test - public void transformOutboundIndex() throws Exception { - Index result = databaseMapping.transformOutboundIndex(index); - assertThat(result, is(sameInstance(index))); - assertThat(result.getDbName(), is(OUT_DB_NAME)); - } - - @Test - public void transformInboundIndex() throws Exception { - Index result = databaseMapping.transformInboundIndex(index); - assertThat(result, is(sameInstance(index))); - assertThat(result.getDbName(), is(IN_DB_NAME)); - } - @Test public void transformOutboundFunction() throws Exception { Function function = new Function(); @@ -561,17 +543,6 @@ public void transformOutboundPartitionSpecs() throws Exception { assertThat(resultSpec.getDbName(), is(OUT_DB_NAME)); } - @Test - public void transformOutboundIndexes() throws Exception { - List indexes = new ArrayList<>(); - indexes.add(index); - List result = databaseMapping.transformOutboundIndexes(indexes); - assertThat(result, is(sameInstance(indexes))); - Index resultIndex = result.get(0); - assertThat(resultIndex, is(sameInstance(index))); - assertThat(resultIndex.getDbName(), is(OUT_DB_NAME)); - } - @Test public void transformInboundColumnStatistics() throws Exception { ColumnStatistics columnStatistics = new ColumnStatistics(); diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseNameMappingTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseNameMappingTest.java index 1076f146f..a0ef18c6e 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseNameMappingTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/DatabaseNameMappingTest.java @@ -34,7 +34,7 @@ public class DatabaseNameMappingTest { private final MetaStoreMapping metaStoreMapping = new PrefixMapping( - new MetaStoreMappingImpl("pre_", "mapping", null, null, DIRECT, + new MetaStoreMappingImpl("pre_", "mapping", "hive", null, null, DIRECT, 0L, new DefaultMetaStoreFilterHookImpl(new HiveConf()))); @Test diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecoratorTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecoratorTest.java index ad9611c25..fc17f82f7 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecoratorTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingDecoratorTest.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import java.util.Arrays; import java.util.List; import org.apache.hadoop.hive.metastore.api.Database; @@ -32,8 +33,6 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; -import com.beust.jcommander.internal.Lists; - @RunWith(MockitoJUnitRunner.class) public class MetaStoreMappingDecoratorTest { @@ -130,7 +129,7 @@ public void transformOutboundDatabaseNameIsNull() throws Exception { @Test public void transformOutboundDatabaseNameMultiple() throws Exception { - when(metaStoreMapping.transformOutboundDatabaseNameMultiple("db")).thenReturn(Lists.newArrayList("trans_db")); + when(metaStoreMapping.transformOutboundDatabaseNameMultiple("db")).thenReturn(Arrays.asList("trans_db")); List result = decorator.transformOutboundDatabaseNameMultiple("db"); assertThat(result.size(), is(1)); assertThat(result.get(0), is("trans_db")); diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java index a65f893cd..9823562fc 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/MetaStoreMappingImplTest.java @@ -58,9 +58,9 @@ public class MetaStoreMappingImplTest { @Before public void init() { - metaStoreMapping = new MetaStoreMappingImpl(DATABASE_PREFIX, NAME, client, accessControlHandler, DIRECT, LATENCY, + metaStoreMapping = new MetaStoreMappingImpl(DATABASE_PREFIX, NAME,"hive", client, accessControlHandler, DIRECT, LATENCY, new DefaultMetaStoreFilterHookImpl(new HiveConf())); - tunneledMetaStoreMapping = new MetaStoreMappingImpl(DATABASE_PREFIX, NAME, client, accessControlHandler, TUNNELED, + tunneledMetaStoreMapping = new MetaStoreMappingImpl(DATABASE_PREFIX, NAME,"hive", client, accessControlHandler, TUNNELED, LATENCY, new DefaultMetaStoreFilterHookImpl(new HiveConf())); } diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixMappingTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixMappingTest.java index 61c10164f..34d566f4d 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixMappingTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixMappingTest.java @@ -27,7 +27,7 @@ @SuppressWarnings("resource") public class PrefixMappingTest { - private final MetaStoreMapping metaStoreMapping = new MetaStoreMappingImpl("prefix_", "mapping", null, null, DIRECT, + private final MetaStoreMapping metaStoreMapping = new MetaStoreMappingImpl("prefix_", "mapping","hive", null, null, DIRECT, 0L, new DefaultMetaStoreFilterHookImpl(new HiveConf())); @Test diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java index 822ef0e13..b3e30c711 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/model/PrefixingMetastoreFilter.java @@ -20,13 +20,13 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.MetaStoreFilterHook; import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionSpec; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.TableMeta; /** * For testing purposes @@ -49,10 +49,17 @@ public Database filterDatabase(Database dataBase) throws MetaException, NoSuchOb } @Override - public List filterTableNames(String dbName, List tableList) throws MetaException { + public List filterTableNames(String catalog, String dbName, List tableList) throws MetaException { return tableList; } + @Override + public List filterTableMetas(List tableMetas) + throws MetaException + { + return tableMetas; + } + @Override public Table filterTable(Table table) throws MetaException, NoSuchObjectException { setLocationPrefix(table); @@ -91,29 +98,11 @@ public Partition filterPartition(Partition partition) throws MetaException, NoSu } @Override - public List filterPartitionNames(String dbName, String tblName, + public List filterPartitionNames(String catalog, String dbName, String tblName, List partitionNames) throws MetaException { return partitionNames; } - @Override - public Index filterIndex(Index index) throws MetaException, NoSuchObjectException { - setLocationPrefix(index.getSd()); - return index; - } - - @Override - public List filterIndexNames(String dbName, String tblName, List indexList) throws MetaException { - return indexList; - } - - @Override - public List filterIndexes(List indexeList) throws MetaException { - for (Index index: indexeList) { - setLocationPrefix(index.getSd()); - } - return indexeList; - } private void setLocationPrefix(Table table) { setLocationPrefix(table.getSd()); diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java index bb99938ae..fc45cbcb0 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/mapping/service/impl/YamlFederatedMetaStoreStorageTest.java @@ -214,31 +214,33 @@ public void saveFederationWriteFederations() throws Exception { storage.insert(newFederatedInstance); storage.saveFederation(); List lines = Files.readAllLines(file.toPath(), StandardCharsets.UTF_8); - assertThat(lines.size(), is(24)); + assertThat(lines.size(), is(26)); assertThat(lines.get(0), is("primary-meta-store:")); assertThat(lines.get(1), is(" access-control-type: READ_ONLY")); - assertThat(lines.get(2), is(" database-prefix: ''")); - assertThat(lines.get(3), is(" latency: 0")); - assertThat(lines.get(4), is(" name: hcom_1")); - assertThat(lines.get(5), is(" remote-meta-store-uris: thrift://localhost:19083")); - assertThat(lines.get(6), is("federated-meta-stores:")); - assertThat(lines.get(7), is("- access-control-type: READ_ONLY")); - assertThat(lines.get(8), is(" database-name-mapping: {}")); - assertThat(lines.get(9), is(" database-prefix: hcom_2_")); - assertThat(lines.get(10), is(" hive-metastore-filter-hook: filter.hook.class")); - assertThat(lines.get(11), is(" latency: 0")); - assertThat(lines.get(12), is(" mapped-databases:")); - assertThat(lines.get(13), is(" - db1")); - assertThat(lines.get(14), is(" - db2")); - assertThat(lines.get(15), is(" mapped-tables:")); - assertThat(lines.get(16), is(" - database: db1")); - assertThat(lines.get(17), is(" mapped-tables:")); - assertThat(lines.get(18), is(" - tbl1")); - assertThat(lines.get(19), is(" - database: db2")); - assertThat(lines.get(20), is(" mapped-tables:")); - assertThat(lines.get(21), is(" - tbl2")); - assertThat(lines.get(22), is(" name: hcom_2")); - assertThat(lines.get(23), is(" remote-meta-store-uris: thrift://localhost:29083")); + assertThat(lines.get(2), is(" catalog: hive")); + assertThat(lines.get(3), is(" database-prefix: ''")); + assertThat(lines.get(4), is(" latency: 0")); + assertThat(lines.get(5), is(" name: hcom_1")); + assertThat(lines.get(6), is(" remote-meta-store-uris: thrift://localhost:19083")); + assertThat(lines.get(7), is("federated-meta-stores:")); + assertThat(lines.get(8), is("- access-control-type: READ_ONLY")); + assertThat(lines.get(9), is(" catalog: hive")); + assertThat(lines.get(10), is(" database-name-mapping: {}")); + assertThat(lines.get(11), is(" database-prefix: hcom_2_")); + assertThat(lines.get(12), is(" hive-metastore-filter-hook: filter.hook.class")); + assertThat(lines.get(13), is(" latency: 0")); + assertThat(lines.get(14), is(" mapped-databases:")); + assertThat(lines.get(15), is(" - db1")); + assertThat(lines.get(16), is(" - db2")); + assertThat(lines.get(17), is(" mapped-tables:")); + assertThat(lines.get(18), is(" - database: db1")); + assertThat(lines.get(19), is(" mapped-tables:")); + assertThat(lines.get(20), is(" - tbl1")); + assertThat(lines.get(21), is(" - database: db2")); + assertThat(lines.get(22), is(" mapped-tables:")); + assertThat(lines.get(23), is(" - tbl2")); + assertThat(lines.get(24), is(" name: hcom_2")); + assertThat(lines.get(25), is(" remote-meta-store-uris: thrift://localhost:29083")); } @Test @@ -296,30 +298,32 @@ public void savePrimaryWriteFederations() throws Exception { storage.insert(newFederatedInstance("hcom_2", "thrift://localhost:29083")); storage.saveFederation(); List lines = Files.readAllLines(file.toPath(), StandardCharsets.UTF_8); - assertThat(lines.size(), is(23)); + assertThat(lines.size(), is(25)); assertThat(lines.get(0), is("primary-meta-store:")); assertThat(lines.get(1), is(" access-control-type: READ_ONLY")); - assertThat(lines.get(2), is(" database-prefix: ''")); - assertThat(lines.get(3), is(" latency: 0")); - assertThat(lines.get(4), is(" mapped-databases:")); - assertThat(lines.get(5), is(" - db1")); - assertThat(lines.get(6), is(" - db2")); - assertThat(lines.get(7), is(" mapped-tables:")); - assertThat(lines.get(8), is(" - database: db1")); - assertThat(lines.get(9), is(" mapped-tables:")); - assertThat(lines.get(10), is(" - tbl1")); - assertThat(lines.get(11), is(" - database: db2")); - assertThat(lines.get(12), is(" mapped-tables:")); - assertThat(lines.get(13), is(" - tbl2")); - assertThat(lines.get(14), is(" name: hcom_1")); - assertThat(lines.get(15), is(" remote-meta-store-uris: thrift://localhost:19083")); - assertThat(lines.get(16), is("federated-meta-stores:")); - assertThat(lines.get(17), is("- access-control-type: READ_ONLY")); - assertThat(lines.get(18), is(" database-name-mapping: {}")); - assertThat(lines.get(19), is(" database-prefix: hcom_2_")); - assertThat(lines.get(20), is(" latency: 0")); - assertThat(lines.get(21), is(" name: hcom_2")); - assertThat(lines.get(22), is(" remote-meta-store-uris: thrift://localhost:29083")); + assertThat(lines.get(2), is(" catalog: hive")); + assertThat(lines.get(3), is(" database-prefix: ''")); + assertThat(lines.get(4), is(" latency: 0")); + assertThat(lines.get(5), is(" mapped-databases:")); + assertThat(lines.get(6), is(" - db1")); + assertThat(lines.get(7), is(" - db2")); + assertThat(lines.get(8), is(" mapped-tables:")); + assertThat(lines.get(9), is(" - database: db1")); + assertThat(lines.get(10), is(" mapped-tables:")); + assertThat(lines.get(11), is(" - tbl1")); + assertThat(lines.get(12), is(" - database: db2")); + assertThat(lines.get(13), is(" mapped-tables:")); + assertThat(lines.get(14), is(" - tbl2")); + assertThat(lines.get(15), is(" name: hcom_1")); + assertThat(lines.get(16), is(" remote-meta-store-uris: thrift://localhost:19083")); + assertThat(lines.get(17), is("federated-meta-stores:")); + assertThat(lines.get(18), is("- access-control-type: READ_ONLY")); + assertThat(lines.get(19), is(" catalog: hive")); + assertThat(lines.get(20), is(" database-name-mapping: {}")); + assertThat(lines.get(21), is(" database-prefix: hcom_2_")); + assertThat(lines.get(22), is(" latency: 0")); + assertThat(lines.get(23), is(" name: hcom_2")); + assertThat(lines.get(24), is(" remote-meta-store-uris: thrift://localhost:29083")); } private PrimaryMetaStore newPrimaryInstance(String name, String remoteMetaStoreUris) { diff --git a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerTest.java b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerTest.java index 6178d6889..bc4d29fed 100644 --- a/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerTest.java +++ b/waggle-dance-core/src/test/java/com/hotels/bdp/waggledance/server/FederatedHMSHandlerTest.java @@ -87,7 +87,6 @@ import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse; import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; import org.apache.hadoop.hive.metastore.api.HiveObjectRef; -import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.LockComponent; import org.apache.hadoop.hive.metastore.api.LockLevel; import org.apache.hadoop.hive.metastore.api.LockRequest; @@ -114,8 +113,12 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeBag; import org.apache.hadoop.hive.metastore.api.Role; +import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; +import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; +import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; import org.apache.hadoop.hive.metastore.api.ShowCompactRequest; import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; @@ -158,7 +161,7 @@ public class FederatedHMSHandlerTest { @Before public void setUp() throws NoSuchObjectException { - handler = new FederatedHMSHandler(databaseMappingService, notifyingFederationService); + handler = new FederatedHMSHandlerHive3(databaseMappingService, notifyingFederationService); when(databaseMappingService.primaryDatabaseMapping()).thenReturn(primaryMapping); when(databaseMappingService.getDatabaseMappings()).thenReturn(Collections.singletonList(primaryMapping)); when(primaryMapping.getClient()).thenReturn(primaryClient); @@ -1057,75 +1060,6 @@ public void isPartitionMarkedForEvent() throws TException { assertThat(result, is(true)); } - @Test - public void add_index() throws TException { - Index newIndex = new Index(); - newIndex.setDbName(DB_P); - Index inboundIndex = new Index(); - Index outboundIndex = new Index(); - Table newTable = new Table(); - newTable.setDbName(DB_P); - Table inboundTable = new Table(); - - when(primaryMapping.transformInboundIndex(newIndex)).thenReturn(inboundIndex); - when(primaryMapping.transformInboundTable(newTable)).thenReturn(inboundTable); - when(primaryMapping.transformOutboundIndex(outboundIndex)).thenReturn(newIndex); - when(primaryClient.add_index(inboundIndex, inboundTable)).thenReturn(outboundIndex); - - Index result = handler.add_index(newIndex, newTable); - verify(primaryMapping, times(2)).checkWritePermissions(DB_P); - assertThat(result, is(newIndex)); - } - - @Test - public void alter_index() throws TException { - Index newIndex = new Index(); - newIndex.setDbName(DB_P); - Index inboundIndex = new Index(); - when(primaryMapping.transformInboundIndex(newIndex)).thenReturn(inboundIndex); - - handler.alter_index(DB_P, "table", "index", newIndex); - verify(primaryMapping, times(2)).checkWritePermissions(DB_P); - verify(primaryClient).alter_index(DB_P, "table", "index", inboundIndex); - } - - @Test - public void drop_index_by_name() throws TException { - when(primaryClient.drop_index_by_name(DB_P, "table", "index", true)).thenReturn(true); - boolean result = handler.drop_index_by_name(DB_P, "table", "index", true); - verify(primaryMapping).checkWritePermissions(DB_P); - assertThat(result, is(true)); - } - - @Test - public void get_index_by_name() throws TException { - Index index = new Index(); - Index outboundIndex = new Index(); - when(primaryClient.get_index_by_name(DB_P, "table", "index")).thenReturn(index); - when(primaryMapping.transformOutboundIndex(index)).thenReturn(outboundIndex); - Index result = handler.get_index_by_name(DB_P, "table", "index"); - assertThat(result, is(outboundIndex)); - } - - @Test - public void get_indexes() throws TException { - List indexList = Collections.singletonList(new Index()); - List outboundIndexList = Collections.singletonList(new Index()); - when(primaryMapping.transformOutboundIndexes(indexList)).thenReturn(outboundIndexList); - when(primaryClient.get_indexes(DB_P, "table", (short) 2)).thenReturn(indexList); - - List result = handler.get_indexes(DB_P, "table", (short) 2); - assertThat(result, is(outboundIndexList)); - } - - @Test - public void get_index_names() throws TException { - List indexNames = Arrays.asList("name1", "name2"); - when(primaryClient.get_index_names(DB_P, "table", (short) 2)).thenReturn(indexNames); - List result = handler.get_index_names(DB_P, "table", (short) 2); - assertThat(result, is(indexNames)); - } - @Test public void update_table_column_statistics() throws TException { ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(true, DB_P, "table"); @@ -1578,7 +1512,7 @@ public void abort_txns() throws TException { @Test public void add_dynamic_partitions() throws TException { - AddDynamicPartitions request = new AddDynamicPartitions(1, DB_P, "table", Collections.emptyList()); + AddDynamicPartitions request = new AddDynamicPartitions(1,1, DB_P, "table", Collections.emptyList()); AddDynamicPartitions inboundRequest = new AddDynamicPartitions(); when(primaryMapping.transformInboundAddDynamicPartitions(request)).thenReturn(inboundRequest); handler.add_dynamic_partitions(request); @@ -1659,10 +1593,14 @@ public void create_table_with_constraints() throws TException { Table inboundTable = new Table(); List primaryKeys = Collections.emptyList(); List foreignKeys = Collections.emptyList(); + List uniqueConstraints= Collections.emptyList(); + List notNullConstraints = Collections.emptyList(); + List< SQLDefaultConstraint > defaultConstraints = Collections.emptyList(); + List< SQLCheckConstraint > checkConstraints = Collections.emptyList(); when(primaryMapping.transformInboundTable(table)).thenReturn(inboundTable); - handler.create_table_with_constraints(table, primaryKeys, foreignKeys); + handler.create_table_with_constraints(table, primaryKeys, foreignKeys, uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints); verify(primaryMapping).checkWritePermissions(DB_P); - verify(primaryClient).create_table_with_constraints(inboundTable, primaryKeys, foreignKeys); + verify(primaryClient).create_table_with_constraints(inboundTable, primaryKeys, foreignKeys, uniqueConstraints, notNullConstraints, defaultConstraints, checkConstraints); } @Test diff --git a/waggle-dance-integration-tests/src/main/java/com/hotels/bdp/waggledance/WaggleDanceRunner.java b/waggle-dance-integration-tests/src/main/java/com/hotels/bdp/waggledance/WaggleDanceRunner.java index 910059a64..8cc769dbb 100644 --- a/waggle-dance-integration-tests/src/main/java/com/hotels/bdp/waggledance/WaggleDanceRunner.java +++ b/waggle-dance-integration-tests/src/main/java/com/hotels/bdp/waggledance/WaggleDanceRunner.java @@ -15,8 +15,6 @@ */ package com.hotels.bdp.waggledance; -import static org.apache.directory.api.util.Strings.isNotEmpty; - import static com.google.common.base.Preconditions.checkArgument; import java.io.File; @@ -38,6 +36,7 @@ import org.apache.commons.vfs2.FileSystemException; import org.apache.commons.vfs2.FileSystemManager; import org.apache.commons.vfs2.VFS; +import org.apache.logging.log4j.util.Strings; import org.springframework.context.ApplicationContext; import org.yaml.snakeyaml.Yaml; @@ -118,8 +117,8 @@ public Builder configurationProperty(String key, String value) { } public Builder federate(String name, String remoteMetaStoreUris, String... mappableDatabases) { - checkArgument(isNotEmpty(name)); - checkArgument(isNotEmpty(remoteMetaStoreUris)); + checkArgument(Strings.isNotEmpty(name)); + checkArgument(Strings.isNotEmpty(remoteMetaStoreUris)); FederatedMetaStore federatedMetaStore = new FederatedMetaStore(name, remoteMetaStoreUris); federatedMetaStore.setMappedDatabases(Arrays.asList(mappableDatabases)); federatedMetaStore.setLatency(8000L); @@ -146,8 +145,8 @@ public Builder withPrimaryDatabaseNameMappingMap(Map databaseNam } public Builder federate(String name, String remoteMetaStoreUris, List mappedTables, String... mappableDatabases) { - checkArgument(isNotEmpty(name)); - checkArgument(isNotEmpty(remoteMetaStoreUris)); + checkArgument(Strings.isNotEmpty(name)); + checkArgument(Strings.isNotEmpty(remoteMetaStoreUris)); FederatedMetaStore federatedMetaStore = new FederatedMetaStore(name, remoteMetaStoreUris); federatedMetaStore.setMappedDatabases(Arrays.asList(mappableDatabases)); federatedMetaStore.setMappedTables(mappedTables); @@ -162,8 +161,8 @@ public Builder federate( AccessControlType accessControlType, String[] mappableDatabases, String[] writeableDatabaseWhiteList) { - checkArgument(isNotEmpty(name)); - checkArgument(isNotEmpty(remoteMetaStoreUris)); + checkArgument(Strings.isNotEmpty(name)); + checkArgument(Strings.isNotEmpty(remoteMetaStoreUris)); FederatedMetaStore federatedMetaStore = new FederatedMetaStore(name, remoteMetaStoreUris, accessControlType); federatedMetaStore.setMappedDatabases(Arrays.asList(mappableDatabases)); federatedMetaStore.setWritableDatabaseWhiteList(Arrays.asList(writeableDatabaseWhiteList)); @@ -197,8 +196,8 @@ public Builder primary( String remoteMetaStoreUris, AccessControlType accessControlType, String... writableDatabaseWhiteList) { - checkArgument(isNotEmpty(name)); - checkArgument(isNotEmpty(remoteMetaStoreUris)); + checkArgument(Strings.isNotEmpty(name)); + checkArgument(Strings.isNotEmpty(remoteMetaStoreUris)); primaryMetaStore = new PrimaryMetaStore(name, remoteMetaStoreUris, accessControlType, writableDatabaseWhiteList); primaryMetaStore.setLatency(8000L); return this; diff --git a/waggle-dance-integration-tests/src/test/java/com/hotels/bdp/waggledance/WaggleDanceIntegrationTest.java b/waggle-dance-integration-tests/src/test/java/com/hotels/bdp/waggledance/WaggleDanceIntegrationTest.java index 3194598f3..d2b5de85c 100644 --- a/waggle-dance-integration-tests/src/test/java/com/hotels/bdp/waggledance/WaggleDanceIntegrationTest.java +++ b/waggle-dance-integration-tests/src/test/java/com/hotels/bdp/waggledance/WaggleDanceIntegrationTest.java @@ -19,6 +19,7 @@ import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.CoreMatchers.startsWith; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; @@ -44,6 +45,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.FunctionType; @@ -106,9 +108,16 @@ public class WaggleDanceIntegrationTest { public @Rule ServerSocketRule graphite = new ServerSocketRule(); public @Rule TemporaryFolder temporaryFolder = new TemporaryFolder(); - public @Rule ThriftHiveMetaStoreJUnitRule localServer = new ThriftHiveMetaStoreJUnitRule(LOCAL_DATABASE); - public @Rule ThriftHiveMetaStoreJUnitRule remoteServer = new ThriftHiveMetaStoreJUnitRule(REMOTE_DATABASE); - public @Rule ThriftHiveMetaStoreJUnitRule newRemoteServer = new ThriftHiveMetaStoreJUnitRule(); + public static Map hiveConfig = new HashMap<>(); + static{ + hiveConfig.put("hive.in.test","true"); + hiveConfig.put("metastore.client.capability.check","false"); + ObjectStore.setTwoMetastoreTesting(true); + } + + public @Rule ThriftHiveMetaStoreJUnitRule localServer = new ThriftHiveMetaStoreJUnitRule(LOCAL_DATABASE,hiveConfig); + public @Rule ThriftHiveMetaStoreJUnitRule remoteServer = new ThriftHiveMetaStoreJUnitRule(REMOTE_DATABASE,hiveConfig); + public @Rule ThriftHiveMetaStoreJUnitRule newRemoteServer = new ThriftHiveMetaStoreJUnitRule("test_database",hiveConfig); public @Rule DataFolder dataFolder = new ClassDataFolder(); private ExecutorService executor; @@ -1006,11 +1015,8 @@ public void manualModeDatabaseNameMapping() throws Exception { List allDatabases = proxy.getAllDatabases(); assertThat(allDatabases.size(), is(5)); - assertThat(allDatabases.get(0), is("default")); - assertThat(allDatabases.get(1), is(LOCAL_DATABASE)); - assertThat(allDatabases.get(2), is("abc")); - assertThat(allDatabases.get(3), is(REMOTE_DATABASE)); - assertThat(allDatabases.get(4), is("xyz")); + String[] expected = new String[]{"default", LOCAL_DATABASE, "abc", REMOTE_DATABASE, "xyz"}; + assertThat("List equality without order", allDatabases, containsInAnyOrder(expected)); // Local table Table waggledLocalTable = proxy.getTable("abc", LOCAL_TABLE); assertNotNull(waggledLocalTable);