Skip to content

Commit

Permalink
Fix half of the tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@dlpzx
dlpzx committed Dec 3, 2024
1 parent cb9fbd0 commit 851f36a
Show file tree
Hide file tree
Showing 9 changed files with 82 additions and 79 deletions.
2 changes: 0 additions & 2 deletions backend/dataall/modules/s3_datasets/api/dataset/types.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,6 @@
name='GlueCrawler',
fields=[
gql.Field(name='Name', type=gql.ID),
gql.Field(name='AwsAccountId', type=gql.String),
gql.Field(name='region', type=gql.String),
gql.Field(name='status', type=gql.String),
],
)
2 changes: 0 additions & 2 deletions backend/dataall/modules/s3_datasets/services/dataset_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -430,8 +430,6 @@ def start_crawler(uri: str, data: dict = None):

return {
'Name': dataset.GlueCrawlerName,
'AwsAccountId': dataset.AwsAccountId,
'region': dataset.region,
'status': crawler.get('LastCrawl', {}).get('Status', 'N/A'),
}

Expand Down
2 changes: 0 additions & 2 deletions frontend/src/modules/S3_Datasets/services/startGlueCrawler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@ export const startGlueCrawler = ({ datasetUri, input }) => ({
mutation StartGlueCrawler($datasetUri: String, $input: CrawlerInput) {
startGlueCrawler(datasetUri: $datasetUri, input: $input) {
Name
AwsAccountId
region
status
}
}
Expand Down
29 changes: 15 additions & 14 deletions tests/modules/s3_datasets/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,19 +76,20 @@ def factory(
datasetUri
label
description
AwsAccountId
S3BucketName
GlueDatabaseName
owner
region,
businessOwnerEmail
businessOwnerDelegationEmails
SamlAdminGroupName
GlueCrawlerName
enableExpiration
expirySetting
expiryMinDuration
expiryMaxDuration
restricted {
AwsAccountId
region
KmsAlias
S3BucketName
GlueDatabaseName
IAMDatasetAdminRoleArn
}
tables{
nodes{
tableUri
Expand Down Expand Up @@ -180,11 +181,11 @@ def factory(dataset: S3Dataset, name, username) -> DatasetTable:
label=name,
owner=username,
datasetUri=dataset.datasetUri,
GlueDatabaseName=dataset.GlueDatabaseName,
GlueDatabaseName=dataset.restricted.GlueDatabaseName,
GlueTableName=name,
region=dataset.region,
AWSAccountId=dataset.AwsAccountId,
S3BucketName=dataset.S3BucketName,
region=dataset.restricted.region,
AWSAccountId=dataset.restricted.AwsAccountId,
S3BucketName=dataset.restricted.S3BucketName,
S3Prefix=f'{name}',
)
session.add(table)
Expand Down Expand Up @@ -325,9 +326,9 @@ def factory(dataset: S3Dataset, name, username) -> DatasetStorageLocation:
label=name,
owner=username,
datasetUri=dataset.datasetUri,
S3BucketName=dataset.S3BucketName,
region=dataset.region,
AWSAccountId=dataset.AwsAccountId,
S3BucketName=dataset.restricted.S3BucketName,
region=dataset.restricted.region,
AWSAccountId=dataset.restricted.AwsAccountId,
S3Prefix=f'{name}',
)
session.add(ds_location)
Expand Down
82 changes: 41 additions & 41 deletions tests/modules/s3_datasets/test_dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,25 +69,27 @@ def test_get_dataset(client, dataset1, env_fixture, group):
query GetDataset($datasetUri:String!){
getDataset(datasetUri:$datasetUri){
label
AwsAccountId
description
region
imported
importedS3Bucket
stewards
owners
imported
restricted {
AwsAccountId
region
importedS3Bucket
}
}
}
""",
datasetUri=dataset1.datasetUri,
username='alice',
groups=[group.name],
)
assert response.data.getDataset.AwsAccountId == env_fixture.AwsAccountId
assert response.data.getDataset.region == env_fixture.region
assert response.data.getDataset.restricted.AwsAccountId == env_fixture.AwsAccountId
assert response.data.getDataset.restricted.region == env_fixture.region
assert response.data.getDataset.label == 'dataset1'
assert response.data.getDataset.imported is False
assert response.data.getDataset.importedS3Bucket is False
assert response.data.getDataset.restricted.importedS3Bucket is False


def test_list_datasets(client, dataset1, group):
Expand Down Expand Up @@ -194,8 +196,6 @@ def test_start_crawler(org_fixture, env_fixture, dataset1, client, group, module
mutation StartGlueCrawler($datasetUri:String, $input:CrawlerInput){
startGlueCrawler(datasetUri:$datasetUri,input:$input){
Name
AwsAccountId
region
status
}
}
Expand All @@ -209,7 +209,7 @@ def test_start_crawler(org_fixture, env_fixture, dataset1, client, group, module
'prefix': 'raw',
},
)
assert response.data.startGlueCrawler.Name == dataset1.GlueCrawlerName
assert response.data.Name == dataset1.restricted.GlueCrawlerName


def test_update_dataset_unauthorized(dataset1, client, group):
Expand Down Expand Up @@ -309,9 +309,11 @@ def test_list_dataset_tables(client, dataset1, group):
tableUri
name
label
GlueDatabaseName
GlueTableName
S3Prefix
restricted{
GlueDatabaseName
GlueTableName
S3Prefix
}
}
}
}
Expand Down Expand Up @@ -391,9 +393,11 @@ def test_delete_dataset(client, dataset, env_fixture, org_fixture, db, module_mo
query GetDataset($datasetUri:String!){
getDataset(datasetUri:$datasetUri){
label
AwsAccountId
restricted {
AwsAccountId
region
}
description
region
}
}
""",
Expand Down Expand Up @@ -428,17 +432,15 @@ def test_import_dataset(org_fixture, env_fixture, dataset1, client, group):
mutation importDataset($input:ImportDatasetInput){
importDataset(input:$input){
label
AwsAccountId
region
imported
importedS3Bucket
importedGlueDatabase
importedKmsKey
importedAdminRole
S3BucketName
GlueDatabaseName
IAMDatasetAdminRoleArn
KmsAlias
restricted {
AwsAccountId
region
S3BucketName
GlueDatabaseName
IAMDatasetAdminRoleArn
KmsAlias
}
}
}
""",
Expand All @@ -457,17 +459,13 @@ def test_import_dataset(org_fixture, env_fixture, dataset1, client, group):
},
)
assert response.data.importDataset.label == 'datasetImported'
assert response.data.importDataset.AwsAccountId == env_fixture.AwsAccountId
assert response.data.importDataset.region == env_fixture.region
assert response.data.importDataset.restricted.AwsAccountId == env_fixture.AwsAccountId
assert response.data.importDataset.restricted.region == env_fixture.region
assert response.data.importDataset.imported is True
assert response.data.importDataset.importedS3Bucket is True
assert response.data.importDataset.importedGlueDatabase is True
assert response.data.importDataset.importedKmsKey is True
assert response.data.importDataset.importedAdminRole is True
assert response.data.importDataset.S3BucketName == 'dhimportedbucket'
assert response.data.importDataset.GlueDatabaseName == 'dhimportedGlueDB'
assert response.data.importDataset.KmsAlias == '1234-YYEY'
assert 'dhimportedRole' in response.data.importDataset.IAMDatasetAdminRoleArn
assert response.data.importDataset.restricted.S3BucketName == 'dhimportedbucket'
assert response.data.importDataset.restricted.GlueDatabaseName == 'dhimportedGlueDB'
assert response.data.importDataset.restricted.KmsAlias == '1234-YYEY'
assert 'dhimportedRole' in response.data.importDataset.restricted.IAMDatasetAdminRoleArn


def test_get_dataset_by_prefix(db, env_fixture, org_fixture):
Expand Down Expand Up @@ -512,13 +510,15 @@ def test_stewardship(client, dataset, env_fixture, org_fixture, db, group2, grou
datasetUri
label
description
AwsAccountId
S3BucketName
GlueDatabaseName
restricted {
AwsAccountId
region
KmsAlias
S3BucketName
GlueDatabaseName
IAMDatasetAdminRoleArn
}
owner
region,
businessOwnerEmail
businessOwnerDelegationEmails
SamlAdminGroupName
stewards
Expand Down
6 changes: 3 additions & 3 deletions tests/modules/s3_datasets/test_dataset_glossary.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ def _columns(db, dataset_fixture, table_fixture) -> List[DatasetTableColumn]:
datasetUri=dataset_fixture.datasetUri,
tableUri=table_fixture.tableUri,
label=f'c{i+1}',
AWSAccountId=dataset_fixture.AwsAccountId,
region=dataset_fixture.region,
AWSAccountId=dataset_fixture.restricted.AwsAccountId,
region=dataset_fixture.restricted.region,
GlueTableName='table',
typeName='String',
owner='user',
GlueDatabaseName=dataset_fixture.GlueDatabaseName,
GlueDatabaseName=dataset_fixture.restricted.GlueDatabaseName,
)
session.add(c)
cols.append(c)
Expand Down
8 changes: 4 additions & 4 deletions tests/modules/s3_datasets/test_dataset_location.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,11 @@ def test_manage_dataset_location(client, dataset1, user, group):
query GetDataset($datasetUri:String!){
getDataset(datasetUri:$datasetUri){
label
AwsAccountId
description
region
imported
importedS3Bucket
restricted {
AwsAccountId
region
}
locations{
nodes{
locationUri
Expand Down
8 changes: 5 additions & 3 deletions tests/modules/s3_datasets/test_dataset_table.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,11 @@ def test_list_dataset_tables(client, dataset_fixture):
tableUri
name
label
GlueDatabaseName
GlueTableName
S3Prefix
restricted {
GlueDatabaseName
GlueTableName
S3Prefix
}
}
}
}
Expand Down
22 changes: 14 additions & 8 deletions tests/modules/s3_datasets_shares/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,15 +83,16 @@ def factory(
datasetUri
label
description
AwsAccountId
S3BucketName
GlueDatabaseName
owner
region,
businessOwnerEmail
businessOwnerDelegationEmails
SamlAdminGroupName
GlueCrawlerName
restricted {
AwsAccountId
region
KmsAlias
S3BucketName
GlueDatabaseName
IAMDatasetAdminRoleArn
}
tables{
nodes{
tableUri
Expand Down Expand Up @@ -253,7 +254,12 @@ def dataset_confidential_fixture(env_fixture, org_fixture, dataset, group) -> S3

@pytest.fixture(scope='module')
def table_fixture(db, dataset_fixture, table, group, user):
table1 = table(dataset=dataset_fixture, name='table1', username=user.username)
dataset = dataset_fixture
dataset.GlueDatabaseName = dataset_fixture.restricted.GlueDatabaseName
dataset.region = dataset_fixture.restricted.region
dataset.S3BucketName = dataset_fixture.restricted.S3BucketName
dataset.AwsAccountId = dataset_fixture.restricted.AwsAccountId
table1 = table(dataset=dataset, name='table1', username=user.username)

with db.scoped_session() as session:
ResourcePolicyService.attach_resource_policy(
Expand Down

0 comments on commit 851f36a

Please sign in to comment.