From d78e81aa8e003d03f746bbc7a968e1e8c40e2254 Mon Sep 17 00:00:00 2001 From: Sofia Sazonova Date: Fri, 20 Dec 2024 14:07:26 +0000 Subject: [PATCH] after merge --- .../core/environment/cdk/environment_stack.py | 2 +- .../shares/s3_datasets_shares/conftest.py | 222 +++++++++--------- .../shared_test_functions.py | 2 +- .../test_new_crossacc_s3_share.py | 38 ++- .../test_persistent_crossacc_share.py | 2 +- 5 files changed, 136 insertions(+), 130 deletions(-) diff --git a/backend/dataall/core/environment/cdk/environment_stack.py b/backend/dataall/core/environment/cdk/environment_stack.py index df6c8cc17..85694cda1 100644 --- a/backend/dataall/core/environment/cdk/environment_stack.py +++ b/backend/dataall/core/environment/cdk/environment_stack.py @@ -697,7 +697,7 @@ def create_integration_tests_role(self): ], effect=iam.Effect.ALLOW, resources=[ - f'arn:aws:iam::{self.account}:role/dataall-test-*', + f'arn:aws:iam::{self.account}:role/dataall-test*', f'arn:aws:iam::{self.account}:role/dataall-session*', ], ), diff --git a/tests_new/integration_tests/modules/shares/s3_datasets_shares/conftest.py b/tests_new/integration_tests/modules/shares/s3_datasets_shares/conftest.py index 68467162d..69aae09fe 100644 --- a/tests_new/integration_tests/modules/shares/s3_datasets_shares/conftest.py +++ b/tests_new/integration_tests/modules/shares/s3_datasets_shares/conftest.py @@ -96,13 +96,13 @@ def session_consumption_role_2(client6, group6, persistent_cross_acc_env_1, pers @pytest.fixture(scope='session') def session_share_1( - client5, - client1, - session_cross_acc_env_1, - session_s3_dataset1, - session_s3_dataset1_tables, - session_s3_dataset1_folders, - group5, + client5, + client1, + session_cross_acc_env_1, + session_s3_dataset1, + session_s3_dataset1_tables, + session_s3_dataset1_folders, + group5, ): share1 = create_share_object( client=client5, @@ -122,13 +122,13 @@ def session_share_1( @pytest.fixture(scope='session') def session_share_2( - client5, - client1, - session_cross_acc_env_1, - session_imported_sse_s3_dataset1, - session_imported_sse_s3_dataset1_tables, - session_imported_sse_s3_dataset1_folders, - group5, + client5, + client1, + session_cross_acc_env_1, + session_imported_sse_s3_dataset1, + session_imported_sse_s3_dataset1_tables, + session_imported_sse_s3_dataset1_folders, + group5, ): share2 = create_share_object( client=client5, @@ -149,16 +149,16 @@ def session_share_2( @pytest.fixture(scope='session') def session_share_3( - client6, - client1, - persistent_env1, - persistent_cross_acc_env_1, - updated_persistent_s3_dataset1, - group6, + client6, + client1, + persistent_env1, + persistent_cross_acc_env_1, + persistent_s3_dataset1, + group6, ): share3 = create_share_object( client=client6, - dataset_or_item_params={'datasetUri': updated_persistent_s3_dataset1.datasetUri}, + dataset_or_item_params={'datasetUri': persistent_s3_dataset1.datasetUri}, environmentUri=persistent_cross_acc_env_1.environmentUri, groupUri=group6, principalId=group6, @@ -174,14 +174,14 @@ def session_share_3( @pytest.fixture(scope='session') def session_share_consrole_1( - client5, - client1, - session_cross_acc_env_1, - session_s3_dataset1, - session_s3_dataset1_tables, - session_s3_dataset1_folders, - group5, - session_consumption_role_1, + client5, + client1, + session_cross_acc_env_1, + session_s3_dataset1, + session_s3_dataset1_tables, + session_s3_dataset1_folders, + group5, + session_consumption_role_1, ): share1cr = create_share_object( client=client5, @@ -201,14 +201,14 @@ def session_share_consrole_1( @pytest.fixture(scope='session') def session_share_consrole_2( - client5, - client1, - session_cross_acc_env_1, - session_imported_sse_s3_dataset1, - session_imported_sse_s3_dataset1_tables, - session_imported_sse_s3_dataset1_folders, - group5, - session_consumption_role_1, + client5, + client1, + session_cross_acc_env_1, + session_imported_sse_s3_dataset1, + session_imported_sse_s3_dataset1_tables, + session_imported_sse_s3_dataset1_folders, + group5, + session_consumption_role_1, ): share2cr = create_share_object( client=client5, @@ -229,17 +229,17 @@ def session_share_consrole_2( @pytest.fixture(scope='session') def session_share_consrole_3( - client6, - client1, - persistent_env1, - persistent_cross_acc_env_1, - updated_persistent_s3_dataset1, - group6, - session_consumption_role_2, + client6, + client1, + persistent_env1, + persistent_cross_acc_env_1, + persistent_s3_dataset1, + group6, + session_consumption_role_2, ): share3cr = create_share_object( client=client6, - dataset_or_item_params={'datasetUri': updated_persistent_s3_dataset1.datasetUri}, + dataset_or_item_params={'datasetUri': persistent_s3_dataset1.datasetUri}, environmentUri=persistent_cross_acc_env_1.environmentUri, groupUri=group6, principalId=session_consumption_role_2.consumptionRoleUri, @@ -263,17 +263,17 @@ def session_share_consrole_3( ] ) def new_share_param( - request, - group5, - group6, - client5, - client6, - session_consumption_role_1, - session_consumption_role_2, - session_s3_dataset1, - updated_persistent_s3_dataset1, - session_cross_acc_env_1, - persistent_cross_acc_env_1, + request, + group5, + group6, + client5, + client6, + session_consumption_role_1, + session_consumption_role_2, + session_s3_dataset1, + persistent_s3_dataset1, + session_cross_acc_env_1, + persistent_cross_acc_env_1, ): # return: client, group, dataset, env, principal_id, principal_type share_type, principal_type = request.param if principal_type == 'Group': @@ -283,7 +283,7 @@ def new_share_param( yield ( client6, group6, - updated_persistent_s3_dataset1, + persistent_s3_dataset1, persistent_cross_acc_env_1, group6, principal_type, @@ -302,7 +302,7 @@ def new_share_param( yield ( client6, group6, - updated_persistent_s3_dataset1, + persistent_s3_dataset1, persistent_cross_acc_env_1, session_consumption_role_2.consumptionRoleUri, principal_type, @@ -318,23 +318,25 @@ def new_share_param( ] ) def share_params_main( - request, - group5, - group6, - client5, - client6, - session_share_1, - session_share_consrole_1, - session_share_3, - session_share_consrole_3, - session_s3_dataset1, - updated_persistent_s3_dataset1, - session_cross_acc_env_1_aws_client, - persistent_cross_acc_env_1_aws_client, - persistent_cross_acc_env_1, - session_cross_acc_env_1, - session_consumption_role_2, - session_consumption_role_1, + request, + group5, + group6, + client5, + client6, + session_share_1, + session_share_consrole_1, + session_share_3, + session_share_consrole_3, + session_s3_dataset1, + persistent_s3_dataset1, + session_cross_acc_env_1_aws_client, + persistent_cross_acc_env_1_aws_client, + persistent_cross_acc_env_1, + session_cross_acc_env_1_integration_role_arn, + persistent_cross_acc_env_1_integration_role_arn, + session_cross_acc_env_1, + session_consumption_role_2, + session_consumption_role_1, ): # return: client, group, env_client, role, share, dataset share_type, principal_type = request.param if principal_type == 'Group': @@ -347,6 +349,7 @@ def share_params_main( session_consumption_role_1, session_share_1, session_s3_dataset1, + session_cross_acc_env_1_integration_role_arn, ) if share_type == 'session_persistent_dataset': yield ( @@ -356,7 +359,8 @@ def share_params_main( persistent_cross_acc_env_1_aws_client, session_consumption_role_2, session_share_3, - updated_persistent_s3_dataset1, + persistent_s3_dataset1, + persistent_cross_acc_env_1_integration_role_arn, ) else: @@ -364,19 +368,23 @@ def share_params_main( yield ( client5, group5, + session_cross_acc_env_1, session_cross_acc_env_1_aws_client, session_consumption_role_1, session_share_consrole_1, session_s3_dataset1, + session_cross_acc_env_1_integration_role_arn, ) if share_type == 'session_persistent_dataset': yield ( client6, group6, + persistent_cross_acc_env_1, persistent_cross_acc_env_1_aws_client, session_consumption_role_2, session_share_consrole_3, - updated_persistent_s3_dataset1, + persistent_s3_dataset1, + persistent_cross_acc_env_1_integration_role_arn, ) @@ -391,18 +399,18 @@ def share_params_main( ] ) def share_params_all( - request, - client5, - client6, - session_share_1, - session_share_consrole_1, - session_share_3, - session_share_consrole_3, - session_s3_dataset1, - session_share_2, - session_share_consrole_2, - session_imported_sse_s3_dataset1, - updated_persistent_s3_dataset1, + request, + client5, + client6, + session_share_1, + session_share_consrole_1, + session_share_3, + session_share_consrole_3, + session_s3_dataset1, + session_share_2, + session_share_consrole_2, + session_imported_sse_s3_dataset1, + persistent_s3_dataset1, ): # return client, share, dataset autoapproval, principal_type, share_type = request.param if autoapproval: @@ -415,12 +423,12 @@ def share_params_all( if share_type == 'session_dataset': yield client5, session_share_1, session_s3_dataset1 if share_type == 'session_persistent_dataset': - yield client6, session_share_3, updated_persistent_s3_dataset1 + yield client6, session_share_3, persistent_s3_dataset1 else: if share_type == 'session_dataset': yield client5, session_share_consrole_1, session_s3_dataset1 if share_type == 'session_persistent_dataset': - yield client6, session_share_consrole_3, updated_persistent_s3_dataset1 + yield client6, session_share_consrole_3, persistent_s3_dataset1 # --------------PERSISTENT FIXTURES---------------------------- @@ -450,16 +458,16 @@ def persistent_consumption_role_1(client5, group5, persistent_cross_acc_env_1, p @pytest.fixture(scope='session') def persistent_group_share_1( - client5, - client1, - persistent_env1, - persistent_cross_acc_env_1, - updated_persistent_s3_dataset1, - group5, + client5, + client1, + persistent_env1, + persistent_cross_acc_env_1, + persistent_s3_dataset1, + group5, ): share1 = create_share_object( client=client5, - dataset_or_item_params={'datasetUri': updated_persistent_s3_dataset1.datasetUri}, + dataset_or_item_params={'datasetUri': persistent_s3_dataset1.datasetUri}, environmentUri=persistent_cross_acc_env_1.environmentUri, groupUri=group5, principalId=group5, @@ -482,17 +490,17 @@ def persistent_group_share_1( @pytest.fixture(scope='session') def persistent_role_share_1( - client5, - client1, - persistent_env1, - persistent_cross_acc_env_1, - updated_persistent_s3_dataset1, - group5, - persistent_consumption_role_1, + client5, + client1, + persistent_env1, + persistent_cross_acc_env_1, + persistent_s3_dataset1, + group5, + persistent_consumption_role_1, ): share1 = create_share_object( client=client5, - dataset_or_item_params={'datasetUri': updated_persistent_s3_dataset1.datasetUri}, + dataset_or_item_params={'datasetUri': persistent_s3_dataset1.datasetUri}, environmentUri=persistent_cross_acc_env_1.environmentUri, groupUri=group5, principalId=persistent_consumption_role_1.consumptionRoleUri, @@ -515,7 +523,7 @@ def persistent_role_share_1( @pytest.fixture(params=['Group', 'ConsumptionRole']) def persistent_share_params_main( - request, persistent_cross_acc_env_1, persistent_role_share_1, persistent_group_share_1 + request, persistent_cross_acc_env_1, persistent_role_share_1, persistent_group_share_1 ): if request.param == 'Group': yield persistent_group_share_1, persistent_cross_acc_env_1 diff --git a/tests_new/integration_tests/modules/shares/s3_datasets_shares/shared_test_functions.py b/tests_new/integration_tests/modules/shares/s3_datasets_shares/shared_test_functions.py index b7e101ca3..035d339d6 100644 --- a/tests_new/integration_tests/modules/shares/s3_datasets_shares/shared_test_functions.py +++ b/tests_new/integration_tests/modules/shares/s3_datasets_shares/shared_test_functions.py @@ -170,7 +170,7 @@ def check_share_items_access( f'arn:aws:s3:{dataset.region}:{dataset.AwsAccountId}:accesspoint/{consumption_data.s3AccessPointName}' ) if principal_type == 'Group': - workgroup = athena_client.get_env_work_group(share_environment.label) + workgroup = athena_client.get_work_group(share_environment.label, group) athena_workgroup_output_location = None else: workgroup = 'primary' diff --git a/tests_new/integration_tests/modules/shares/s3_datasets_shares/test_new_crossacc_s3_share.py b/tests_new/integration_tests/modules/shares/s3_datasets_shares/test_new_crossacc_s3_share.py index 030128bed..11d87859e 100644 --- a/tests_new/integration_tests/modules/shares/s3_datasets_shares/test_new_crossacc_s3_share.py +++ b/tests_new/integration_tests/modules/shares/s3_datasets_shares/test_new_crossacc_s3_share.py @@ -137,7 +137,7 @@ def test_reject_share(client1, new_share_param): def test_change_share_purpose(share_params_main): - client, _, _, _, _, share, _ = share_params_main + client, _, _, _, _, share, _, _ = share_params_main change_request_purpose = update_share_request_reason(client, share.shareUri, 'new purpose') assert_that(change_request_purpose).is_true() updated_share = get_share_object(client, share.shareUri) @@ -153,47 +153,45 @@ def test_submit_object(share_params_all): @pytest.mark.dependency(name='share_approved', depends=['share_submitted']) def test_approve_share(client1, share_params_main): - client, _, _, _, _, share, _ = share_params_main + client, _, _, _, _, share, _, _ = share_params_main check_approve_share_object(client1, share.shareUri) @pytest.mark.dependency(name='share_succeeded', depends=['share_approved']) def test_share_succeeded(client1, share_params_main): - client, _, _, _, _, share, _ = share_params_main + client, _, _, _, _, share, _, _ = share_params_main check_share_succeeded(client1, share.shareUri, check_contains_all_item_types=True) @pytest.mark.dependency(name='share_verified', depends=['share_succeeded']) def test_verify_share_items(client1, share_params_main): - client, _, _, _, _, share, _ = share_params_main + client, _, _, _, _, share, _, _ = share_params_main check_verify_share_items(client1, share.shareUri) @pytest.mark.dependency(depends=['share_verified']) def test_check_item_access(share_params_main): - client, group, env, env_client, role, share, _ = share_params_main + client, group, env, env_client, role, share, _, _ = share_params_main check_share_items_access(client, group, share.shareUri, env, role, env_client) @pytest.mark.dependency(name='unhealthy_items', depends=['share_verified']) -def test_unhealthy_items( - client5, session_cross_acc_env_1_aws_client, session_cross_acc_env_1_integration_role_arn, share_params_main -): - client, _, _, _, _, share, _ = share_params_main - iam = session_cross_acc_env_1_aws_client.resource('iam') +def test_unhealthy_items(share_params_main): + client, group, env, env_client, role, share, _, integration_role_arn = share_params_main + iam = env_client.resource('iam') principal_role = iam.Role(share.principal.principalRoleName) # break s3 by removing policies for policy in principal_role.attached_policies.all(): if '/dataall-env-' in policy.arn and 'share-policy' in policy.arn: principal_role.detach_policy(PolicyArn=policy.arn) # break lf by removing DESCRIBE perms from principal - lf_client = LakeFormationClient(session_cross_acc_env_1_aws_client, session_cross_acc_env_1_aws_client.region_name) - lf_client.add_role_to_datalake_admin(session_cross_acc_env_1_integration_role_arn) + lf_client = LakeFormationClient(env_client, env_client.region_name) + lf_client.add_role_to_datalake_admin(integration_role_arn) db_name = f'dataall_{share.dataset.datasetName}_{share.dataset.datasetUri}_shared'.replace('-', '_') lf_client.revoke_db_perms(principal_role.arn, db_name, ['DESCRIBE']) # verify all items are `Unhealthy` check_verify_share_items( - client5, + client, share.shareUri, expected_health_status=['Unhealthy'], expected_health_msg=[ @@ -204,19 +202,19 @@ def test_unhealthy_items( @pytest.mark.dependency(depends=['share_approved']) -def test_reapply_unauthoried(client5, share_params_main): - share, _, _ = share_params_main +def test_reapply_unauthoried(share_params_main): + client, _, _, _, _, share, _, _ = share_params_main share_uri = share.shareUri - share_object = get_share_object(client5, share_uri) + share_object = get_share_object(client, share_uri) item_uris = [item.shareItemUri for item in share_object['items'].nodes] - assert_that(reapply_items_share_object).raises(GqlError).when_called_with(client5, share_uri, item_uris).contains( + assert_that(reapply_items_share_object).raises(GqlError).when_called_with(client, share_uri, item_uris).contains( 'UnauthorizedOperation' ) @pytest.mark.dependency(depends=['share_approved']) def test_reapply(client1, share_params_main): - share, _, _ = share_params_main + _, _, _, _, _, share, _, _ = share_params_main share_uri = share.shareUri share_object = get_share_object(client1, share_uri) item_uris = [item.shareItemUri for item in share_object['items'].nodes] @@ -229,14 +227,14 @@ def test_reapply(client1, share_params_main): @pytest.mark.dependency(name='share_revoked', depends=['share_succeeded']) def test_revoke_share(share_params_main): - client, _,_,_,_, share, _ = share_params_main + client, _, _, _, _, share, _, _ = share_params_main check_share_ready(client, share.shareUri) revoke_and_check_all_shared_items(client, share.shareUri, check_contains_all_item_types=True) @pytest.mark.dependency(name='share_revoke_succeeded', depends=['share_revoked']) def test_revoke_succeeded(client1, share_params_main): - client, group, env, env_client, role, share, dataset = share_params_main + client, group, env, env_client, role, share, dataset, _ = share_params_main check_all_items_revoke_job_succeeded(client, share.shareUri, check_contains_all_item_types=True) check_share_items_access( client, diff --git a/tests_new/integration_tests/modules/shares/s3_datasets_shares/test_persistent_crossacc_share.py b/tests_new/integration_tests/modules/shares/s3_datasets_shares/test_persistent_crossacc_share.py index ad5a09f4f..1e2e7683d 100644 --- a/tests_new/integration_tests/modules/shares/s3_datasets_shares/test_persistent_crossacc_share.py +++ b/tests_new/integration_tests/modules/shares/s3_datasets_shares/test_persistent_crossacc_share.py @@ -99,7 +99,7 @@ def test_re_share_succeeded( check_verify_share_items(client5, share.shareUri) check_share_items_access( client5, - share.group, + share.principal.SamlGroupName, share.shareUri, env, persistent_consumption_role_1,