From e847d8acf93d90f23316f2c5280998b19d907840 Mon Sep 17 00:00:00 2001 From: NikaHsn Date: Fri, 20 Sep 2024 12:57:15 -0700 Subject: [PATCH 01/25] feat(core): add storage bucket type (#5478) --- .../types/exception/amplify_exception.dart | 1 + .../invalid_storage_bucket_exception.dart | 15 +++++++++ .../lib/src/types/storage/bucket_info.dart | 9 ++++++ .../lib/src/types/storage/storage_bucket.dart | 19 ++++++++++++ .../storage/storage_bucket_from_outputs.dart | 31 +++++++++++++++++++ .../lib/src/types/storage/storage_types.dart | 2 ++ 6 files changed, 77 insertions(+) create mode 100644 packages/amplify_core/lib/src/types/exception/storage/invalid_storage_bucket_exception.dart create mode 100644 packages/amplify_core/lib/src/types/storage/bucket_info.dart create mode 100644 packages/amplify_core/lib/src/types/storage/storage_bucket.dart create mode 100644 packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart diff --git a/packages/amplify_core/lib/src/types/exception/amplify_exception.dart b/packages/amplify_core/lib/src/types/exception/amplify_exception.dart index aedc1c1e19..a60f435bd6 100644 --- a/packages/amplify_core/lib/src/types/exception/amplify_exception.dart +++ b/packages/amplify_core/lib/src/types/exception/amplify_exception.dart @@ -21,6 +21,7 @@ part 'network_exception.dart'; part 'push/push_notification_exception.dart'; part 'storage/access_denied_exception.dart'; part 'storage/http_status_exception.dart'; +part 'storage/invalid_storage_bucket_exception.dart'; part 'storage/local_file_not_found_exception.dart'; part 'storage/not_found_exception.dart'; part 'storage/operation_canceled_exception.dart'; diff --git a/packages/amplify_core/lib/src/types/exception/storage/invalid_storage_bucket_exception.dart b/packages/amplify_core/lib/src/types/exception/storage/invalid_storage_bucket_exception.dart new file mode 100644 index 0000000000..5e5c56fe0e --- /dev/null +++ b/packages/amplify_core/lib/src/types/exception/storage/invalid_storage_bucket_exception.dart @@ -0,0 +1,15 @@ +part of '../amplify_exception.dart'; + +/// {@template amplify_core.storage.invalid_storage_bucket_exception} +/// Exception thrown when the [StorageBucket] is invalid. +/// {@endtemplate} +class InvalidStorageBucketException extends StorageException { + const InvalidStorageBucketException( + super.message, { + super.recoverySuggestion, + super.underlyingException, + }); + + @override + String get runtimeTypeName => 'InvalidStorageBucketException'; +} diff --git a/packages/amplify_core/lib/src/types/storage/bucket_info.dart b/packages/amplify_core/lib/src/types/storage/bucket_info.dart new file mode 100644 index 0000000000..a3c065ef23 --- /dev/null +++ b/packages/amplify_core/lib/src/types/storage/bucket_info.dart @@ -0,0 +1,9 @@ +/// {@template amplify_core.storage.bucket_info} +/// Presents a storage bucket information. +/// {@endtemplate} +class BucketInfo { + /// {@macro amplify_core.storage.bucket_info} + const BucketInfo({required this.bucketName, required this.region}); + final String bucketName; + final String region; +} diff --git a/packages/amplify_core/lib/src/types/storage/storage_bucket.dart b/packages/amplify_core/lib/src/types/storage/storage_bucket.dart new file mode 100644 index 0000000000..711e30d18b --- /dev/null +++ b/packages/amplify_core/lib/src/types/storage/storage_bucket.dart @@ -0,0 +1,19 @@ +import 'package:amplify_core/src/config/amplify_outputs/storage/storage_outputs.dart'; +import 'package:amplify_core/src/types/storage/bucket_info.dart'; +import 'package:amplify_core/src/types/storage/storage_bucket_from_outputs.dart'; +import 'package:meta/meta.dart'; + +/// Presents a storage bucket. +class StorageBucket { + /// Creates a [StorageBucket] from [BucketInfo]. + const StorageBucket.fromBucketInfo(this._info); + + /// Creates a [StorageBucket] defined by the [name] in AmplifyOutputs file. + factory StorageBucket.fromOutputs(String name) => + StorageBucketFromOutputs(name); + + final BucketInfo _info; + + @internal + BucketInfo resolveBucketInfo(StorageOutputs? storageOutputs) => _info; +} diff --git a/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart b/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart new file mode 100644 index 0000000000..4ee6cedb83 --- /dev/null +++ b/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart @@ -0,0 +1,31 @@ +import 'package:amplify_core/amplify_core.dart'; +import 'package:amplify_core/src/config/amplify_outputs/storage/storage_outputs.dart'; +import 'package:meta/meta.dart'; + +/// {@template amplify_core.storage.storage_bucket_from_outputs} +/// Creates a [StorageBucket] defined by the name in AmplifyOutputs file. +/// {@endtemplate} +@internal +class StorageBucketFromOutputs implements StorageBucket { + /// {@macro amplify_core.storage.storage_bucket_from_outputs} + const StorageBucketFromOutputs(this._name); + + final String _name; + + @override + BucketInfo resolveBucketInfo(StorageOutputs? storageOutputs) { + assert( + storageOutputs != null, + const InvalidStorageBucketException( + 'Amplify Storage is not configured.', + recoverySuggestion: + 'Make sure storage exists in the Amplify Outputs file.', + ), + ); + // TODO(nikahsn): fix after adding buckets to StorageOutputs. + return BucketInfo( + bucketName: _name, + region: storageOutputs!.awsRegion, + ); + } +} diff --git a/packages/amplify_core/lib/src/types/storage/storage_types.dart b/packages/amplify_core/lib/src/types/storage/storage_types.dart index 9324a72683..1e4bfbfb02 100644 --- a/packages/amplify_core/lib/src/types/storage/storage_types.dart +++ b/packages/amplify_core/lib/src/types/storage/storage_types.dart @@ -11,6 +11,7 @@ export '../exception/amplify_exception.dart' StorageOperationCanceledException, NetworkException, UnknownException; +export 'bucket_info.dart'; export 'copy_operation.dart'; export 'copy_options.dart'; export 'copy_request.dart'; @@ -44,6 +45,7 @@ export 'remove_operation.dart'; export 'remove_options.dart'; export 'remove_request.dart'; export 'remove_result.dart'; +export 'storage_bucket.dart'; export 'storage_item.dart'; export 'storage_path.dart'; export 'transfer_progress.dart'; From 3fecfa4d876b936b374b825a0e7eab00827efe67 Mon Sep 17 00:00:00 2001 From: ekjotmultani Date: Tue, 17 Sep 2024 10:35:48 -0700 Subject: [PATCH 02/25] updating storage_outputs class for multi-bucket support --- .../storage/storage_outputs.dart | 5 +++- .../storage/storage_outputs.g.dart | 25 +++++++++++++++---- .../config/amplify_outputs/test_data.dart | 23 ++++++++++++++--- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart index 9db9356aa6..416a658cda 100644 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart @@ -12,7 +12,7 @@ part 'storage_outputs.g.dart'; class StorageOutputs with AWSEquatable, AWSSerializable, AWSDebuggable { /// {@macro amplify_core.amplify_outputs.storage_outputs} - const StorageOutputs({required this.awsRegion, required this.bucketName}); + const StorageOutputs({required this.awsRegion, required this.bucketName, this.buckets}); factory StorageOutputs.fromJson(Map json) => _$StorageOutputsFromJson(json); @@ -23,6 +23,9 @@ class StorageOutputs /// The Amazon S3 bucket name. final String bucketName; + /// The list of buckets if there are multiple buckets for the project + final List>? buckets; + @override List get props => [awsRegion, bucketName]; diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart index 7b90421189..8b77e74549 100644 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart @@ -16,6 +16,11 @@ StorageOutputs _$StorageOutputsFromJson(Map json) => final val = StorageOutputs( awsRegion: $checkedConvert('aws_region', (v) => v as String), bucketName: $checkedConvert('bucket_name', (v) => v as String), + buckets: $checkedConvert( + 'buckets', + (v) => (v as List?) + ?.map((e) => Map.from(e as Map)) + .toList()), ); return val; }, @@ -25,8 +30,18 @@ StorageOutputs _$StorageOutputsFromJson(Map json) => }, ); -Map _$StorageOutputsToJson(StorageOutputs instance) => - { - 'aws_region': instance.awsRegion, - 'bucket_name': instance.bucketName, - }; +Map _$StorageOutputsToJson(StorageOutputs instance) { + final val = { + 'aws_region': instance.awsRegion, + 'bucket_name': instance.bucketName, + }; + + void writeNotNull(String key, dynamic value) { + if (value != null) { + val[key] = value; + } + } + + writeNotNull('buckets', instance.buckets); + return val; +} diff --git a/packages/amplify_core/test/config/amplify_outputs/test_data.dart b/packages/amplify_core/test/config/amplify_outputs/test_data.dart index 317e7a78b4..4a1f5e7b87 100644 --- a/packages/amplify_core/test/config/amplify_outputs/test_data.dart +++ b/packages/amplify_core/test/config/amplify_outputs/test_data.dart @@ -99,9 +99,26 @@ const amplifyoutputs = '''{ ] }, "storage": { - "aws_region": "oem dks", - "bucket_name": "dolor et esse" - }, + "aws_region": "us-east-2", + "bucket_name": "amplifyTeamDrive-one-stora-testbucketgen2bucket0b8c-9ggcfqfunkjr", + "buckets": [ + { + "name": "amplifyTeamDrive-one", + "bucket_name": "amplifyTeamDrive-one-stora-testbucketgen2bucket0b8c-9ggcfqfunkjr", + "aws_region": "us-east-2" + }, + { + "name": "amplifyTeamDrive-two", + "bucket_name": "amplifyTeamDrive-two-stora-testbucketgen2bucket0b8c-2", + "aws_region": "us-east-2" + }, + { + "name": "amplifyTeamDrive-three", + "bucket_name": "amplifyTeamDrive-three-stora-testbucketgen2bucket0b8c-3", + "aws_region": "us-east-2" + } + ] +}, "version": "1" } '''; From 10aee0ba3ac4883c8de58202aa5b59070dcbf04b Mon Sep 17 00:00:00 2001 From: ekjotmultani Date: Tue, 17 Sep 2024 14:50:21 -0700 Subject: [PATCH 03/25] seperated bucket out into its own class instead of using a map --- .../storage/storage_output_bucket.dart | 16 ++++++++++++++++ .../amplify_outputs/storage/storage_outputs.dart | 8 +++++--- .../storage/storage_outputs.g.dart | 5 +++-- 3 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_output_bucket.dart diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_output_bucket.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_output_bucket.dart new file mode 100644 index 0000000000..66cc099dbc --- /dev/null +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_output_bucket.dart @@ -0,0 +1,16 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + + + +/// {@template amplify_core.amplify_outputs.amazon_pinpoint_channel} +/// Supported channels for Amazon Pinpoint. +/// {@endtemplate} +class StorageOutputBucket { + StorageOutputBucket(this.name, this.bucketName, this.awsRegion); + factory StorageOutputBucket.fromJson(Map json) => StorageOutputBucket(json['name'].toString(), json['bucket_name'].toString(), json['aws_region'].toString()); + String name; + String bucketName; + String awsRegion; + Map toJson() => {'name':name, 'bucket_name':bucketName, 'aws_region':awsRegion}; +} diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart index 416a658cda..f6744287ac 100644 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import 'package:amplify_core/amplify_core.dart'; +import 'package:amplify_core/src/config/amplify_outputs/storage/storage_output_bucket.dart'; part 'storage_outputs.g.dart'; @@ -14,6 +15,7 @@ class StorageOutputs /// {@macro amplify_core.amplify_outputs.storage_outputs} const StorageOutputs({required this.awsRegion, required this.bucketName, this.buckets}); + factory StorageOutputs.fromJson(Map json) => _$StorageOutputsFromJson(json); @@ -24,10 +26,10 @@ class StorageOutputs final String bucketName; /// The list of buckets if there are multiple buckets for the project - final List>? buckets; - + final List? buckets; + @override - List get props => [awsRegion, bucketName]; + List get props => [awsRegion, bucketName, buckets]; @override String get runtimeTypeName => 'StorageOutputs'; diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart index 8b77e74549..2fb3499954 100644 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart @@ -19,7 +19,8 @@ StorageOutputs _$StorageOutputsFromJson(Map json) => buckets: $checkedConvert( 'buckets', (v) => (v as List?) - ?.map((e) => Map.from(e as Map)) + ?.map((e) => + StorageOutputBucket.fromJson(e as Map)) .toList()), ); return val; @@ -42,6 +43,6 @@ Map _$StorageOutputsToJson(StorageOutputs instance) { } } - writeNotNull('buckets', instance.buckets); + writeNotNull('buckets', instance.buckets?.map((e) => e.toJson()).toList()); return val; } From 95f3e55d3910414e6218c1eb6312d97413046002 Mon Sep 17 00:00:00 2001 From: ekjotmultani Date: Wed, 18 Sep 2024 11:31:25 -0700 Subject: [PATCH 04/25] made bucket_output class a proper output class like the others in amplify-flutter --- packages/amplify_core/doc/lib/auth.dart | 6 +-- .../storage/bucket_output.dart | 37 +++++++++++++++++++ .../storage/bucket_output.g.dart | 34 +++++++++++++++++ .../storage/storage_output_bucket.dart | 16 -------- .../storage/storage_outputs.dart | 4 +- .../storage/storage_outputs.g.dart | 3 +- 6 files changed, 77 insertions(+), 23 deletions(-) create mode 100644 packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.dart create mode 100644 packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.g.dart delete mode 100644 packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_output_bucket.dart diff --git a/packages/amplify_core/doc/lib/auth.dart b/packages/amplify_core/doc/lib/auth.dart index d4f33ab4c7..2a56b43f23 100644 --- a/packages/amplify_core/doc/lib/auth.dart +++ b/packages/amplify_core/doc/lib/auth.dart @@ -110,7 +110,7 @@ Future _handleSignInResult(SignInResult result) async { // #enddocregion handle-signin, handle-confirm-signin-sms, handle-confirm-signin-new-password, handle-confirm-signin-custom-challenge, handle-confirm-signin-reset-password, handle-confirm-signin-confirm-signup, handle-confirm-signin-done, handle-confirm-signin-mfa-selection, handle-confirm-signin-totp-setup, handle-confirm-signin-totp-code, handle-confirm-signin-email-code, handle-confirm-signin-mfa-setup-selection, handle-confirm-signin-email-setup // #docregion handle-confirm-signin-mfa-selection case AuthSignInStep.continueSignInWithMfaSelection: - final allowedMfaTypes = result.nextStep.allowedMfaTypes!; + final allowedMfaTypes = result.nextStep.allowedMfaTypes; final selection = await _promptUserPreference(allowedMfaTypes); return _handleMfaSelection(selection); // #enddocregion handle-confirm-signin-mfa-selection @@ -126,7 +126,7 @@ Future _handleSignInResult(SignInResult result) async { // #enddocregion handle-confirm-signin-mfa-setup-selection // #docregion handle-confirm-signin-totp-setup case AuthSignInStep.continueSignInWithTotpSetup: - final totpSetupDetails = result.nextStep.totpSetupDetails!; + final totpSetupDetails = result.nextStep.totpSetupDetails; final setupUri = totpSetupDetails.getSetupUri(appName: 'MyApp'); safePrint('Open URI to complete setup: $setupUri'); // #enddocregion handle-confirm-signin-totp-setup @@ -276,7 +276,7 @@ Future signOutGlobally() async { if (result is CognitoCompleteSignOut) { safePrint('Sign out completed successfully'); } else if (result is CognitoPartialSignOut) { - final globalSignOutException = result.globalSignOutException!; + final globalSignOutException = result.globalSignOutException; final accessToken = globalSignOutException.accessToken; // Retry the global sign out using the access token, if desired // ... diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.dart new file mode 100644 index 0000000000..9c09884ee3 --- /dev/null +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.dart @@ -0,0 +1,37 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import 'package:amplify_core/amplify_core.dart'; + +part 'bucket_output.g.dart'; + +@zAmplifyOutputsSerializable +class BucketOutput + with AWSEquatable, AWSSerializable, AWSDebuggable{ + + const BucketOutput({required this.name, required this.bucketName, required this.awsRegion}); + + factory BucketOutput.fromJson(Map json) => + _$BucketOutputFromJson(json); + + final String name; + + final String bucketName; + + final String awsRegion; + + @override + List get props => [ + name, + bucketName, + awsRegion, + ]; + + @override + String get runtimeTypeName => 'BucketOutput'; + + @override + Object? toJson() { + return _$BucketOutputToJson(this); + } +} diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.g.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.g.dart new file mode 100644 index 0000000000..331517e83e --- /dev/null +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.g.dart @@ -0,0 +1,34 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +// ignore_for_file: deprecated_member_use_from_same_package + +part of 'bucket_output.dart'; + +// ************************************************************************** +// JsonSerializableGenerator +// ************************************************************************** + +BucketOutput _$BucketOutputFromJson(Map json) => + $checkedCreate( + 'BucketOutput', + json, + ($checkedConvert) { + final val = BucketOutput( + name: $checkedConvert('name', (v) => v as String), + bucketName: $checkedConvert('bucket_name', (v) => v as String), + awsRegion: $checkedConvert('aws_region', (v) => v as String), + ); + return val; + }, + fieldKeyMap: const { + 'bucketName': 'bucket_name', + 'awsRegion': 'aws_region' + }, + ); + +Map _$BucketOutputToJson(BucketOutput instance) => + { + 'name': instance.name, + 'bucket_name': instance.bucketName, + 'aws_region': instance.awsRegion, + }; diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_output_bucket.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_output_bucket.dart deleted file mode 100644 index 66cc099dbc..0000000000 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_output_bucket.dart +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - - - -/// {@template amplify_core.amplify_outputs.amazon_pinpoint_channel} -/// Supported channels for Amazon Pinpoint. -/// {@endtemplate} -class StorageOutputBucket { - StorageOutputBucket(this.name, this.bucketName, this.awsRegion); - factory StorageOutputBucket.fromJson(Map json) => StorageOutputBucket(json['name'].toString(), json['bucket_name'].toString(), json['aws_region'].toString()); - String name; - String bucketName; - String awsRegion; - Map toJson() => {'name':name, 'bucket_name':bucketName, 'aws_region':awsRegion}; -} diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart index f6744287ac..9a0fc949de 100644 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import 'package:amplify_core/amplify_core.dart'; -import 'package:amplify_core/src/config/amplify_outputs/storage/storage_output_bucket.dart'; +import 'package:amplify_core/src/config/amplify_outputs/storage/bucket_output.dart'; part 'storage_outputs.g.dart'; @@ -26,7 +26,7 @@ class StorageOutputs final String bucketName; /// The list of buckets if there are multiple buckets for the project - final List? buckets; + final List? buckets; @override List get props => [awsRegion, bucketName, buckets]; diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart index 2fb3499954..9dac1f7c12 100644 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart @@ -19,8 +19,7 @@ StorageOutputs _$StorageOutputsFromJson(Map json) => buckets: $checkedConvert( 'buckets', (v) => (v as List?) - ?.map((e) => - StorageOutputBucket.fromJson(e as Map)) + ?.map((e) => BucketOutput.fromJson(e as Map)) .toList()), ); return val; From 06cad0ad880a1132e3e2042888efb7634dd2ca8f Mon Sep 17 00:00:00 2001 From: ekjotmultani Date: Wed, 18 Sep 2024 12:19:27 -0700 Subject: [PATCH 05/25] added doc comments and changed name of bucket class --- devtools_options.yaml | 3 ++ .../storage/bucket_output.dart | 37 ----------------- .../storage/bucket_outputs.dart | 41 +++++++++++++++++++ ...et_output.g.dart => bucket_outputs.g.dart} | 10 ++--- .../storage/storage_outputs.dart | 4 +- .../storage/storage_outputs.g.dart | 3 +- 6 files changed, 53 insertions(+), 45 deletions(-) create mode 100644 devtools_options.yaml delete mode 100644 packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.dart create mode 100644 packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_outputs.dart rename packages/amplify_core/lib/src/config/amplify_outputs/storage/{bucket_output.g.dart => bucket_outputs.g.dart} (79%) diff --git a/devtools_options.yaml b/devtools_options.yaml new file mode 100644 index 0000000000..fa0b357c4f --- /dev/null +++ b/devtools_options.yaml @@ -0,0 +1,3 @@ +description: This file stores settings for Dart & Flutter DevTools. +documentation: https://docs.flutter.dev/tools/devtools/extensions#configure-extension-enablement-states +extensions: diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.dart deleted file mode 100644 index 9c09884ee3..0000000000 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.dart +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -import 'package:amplify_core/amplify_core.dart'; - -part 'bucket_output.g.dart'; - -@zAmplifyOutputsSerializable -class BucketOutput - with AWSEquatable, AWSSerializable, AWSDebuggable{ - - const BucketOutput({required this.name, required this.bucketName, required this.awsRegion}); - - factory BucketOutput.fromJson(Map json) => - _$BucketOutputFromJson(json); - - final String name; - - final String bucketName; - - final String awsRegion; - - @override - List get props => [ - name, - bucketName, - awsRegion, - ]; - - @override - String get runtimeTypeName => 'BucketOutput'; - - @override - Object? toJson() { - return _$BucketOutputToJson(this); - } -} diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_outputs.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_outputs.dart new file mode 100644 index 0000000000..c3e05b87ec --- /dev/null +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_outputs.dart @@ -0,0 +1,41 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import 'package:amplify_core/amplify_core.dart'; + +part 'bucket_outputs.g.dart'; + +/// {@template amplify_core.amplify_outputs.bucket_outputs} +/// The Amplify Gen 2 outputs for Buckets in the Storage category. +/// {@endtemplate} +@zAmplifyOutputsSerializable +class BucketOutputs + with AWSEquatable, AWSSerializable, AWSDebuggable{ + /// {@macro amplify_core.amplify_outputs.bucket_outputs} + const BucketOutputs({required this.name, required this.bucketName, required this.awsRegion,}); + + factory BucketOutputs.fromJson(Map json) => + _$BucketOutputsFromJson(json); + + /// The user friendly name of the bucket + final String name; + /// The Amazon S3 bucket name. + final String bucketName; + /// The AWS region of Amazon S3 resources. + final String awsRegion; + + @override + List get props => [ + name, + bucketName, + awsRegion, + ]; + + @override + String get runtimeTypeName => 'BucketOutputs'; + + @override + Object? toJson() { + return _$BucketOutputsToJson(this); + } +} diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.g.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_outputs.g.dart similarity index 79% rename from packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.g.dart rename to packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_outputs.g.dart index 331517e83e..0d60c85fcc 100644 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_output.g.dart +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_outputs.g.dart @@ -2,18 +2,18 @@ // ignore_for_file: deprecated_member_use_from_same_package -part of 'bucket_output.dart'; +part of 'bucket_outputs.dart'; // ************************************************************************** // JsonSerializableGenerator // ************************************************************************** -BucketOutput _$BucketOutputFromJson(Map json) => +BucketOutputs _$BucketOutputsFromJson(Map json) => $checkedCreate( - 'BucketOutput', + 'BucketOutputs', json, ($checkedConvert) { - final val = BucketOutput( + final val = BucketOutputs( name: $checkedConvert('name', (v) => v as String), bucketName: $checkedConvert('bucket_name', (v) => v as String), awsRegion: $checkedConvert('aws_region', (v) => v as String), @@ -26,7 +26,7 @@ BucketOutput _$BucketOutputFromJson(Map json) => }, ); -Map _$BucketOutputToJson(BucketOutput instance) => +Map _$BucketOutputsToJson(BucketOutputs instance) => { 'name': instance.name, 'bucket_name': instance.bucketName, diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart index 9a0fc949de..87f34ad570 100644 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import 'package:amplify_core/amplify_core.dart'; -import 'package:amplify_core/src/config/amplify_outputs/storage/bucket_output.dart'; +import 'package:amplify_core/src/config/amplify_outputs/storage/bucket_outputs.dart'; part 'storage_outputs.g.dart'; @@ -26,7 +26,7 @@ class StorageOutputs final String bucketName; /// The list of buckets if there are multiple buckets for the project - final List? buckets; + final List? buckets; @override List get props => [awsRegion, bucketName, buckets]; diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart index 9dac1f7c12..40d147f387 100644 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.g.dart @@ -19,7 +19,8 @@ StorageOutputs _$StorageOutputsFromJson(Map json) => buckets: $checkedConvert( 'buckets', (v) => (v as List?) - ?.map((e) => BucketOutput.fromJson(e as Map)) + ?.map( + (e) => BucketOutputs.fromJson(e as Map)) .toList()), ); return val; From 565c46dbb5ee401600b285547fc697562752c24a Mon Sep 17 00:00:00 2001 From: ekjotmultani <43255916+ekjotmultani@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:42:38 -0700 Subject: [PATCH 06/25] Update auth.dart --- packages/amplify_core/doc/lib/auth.dart | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/amplify_core/doc/lib/auth.dart b/packages/amplify_core/doc/lib/auth.dart index 2a56b43f23..d4f33ab4c7 100644 --- a/packages/amplify_core/doc/lib/auth.dart +++ b/packages/amplify_core/doc/lib/auth.dart @@ -110,7 +110,7 @@ Future _handleSignInResult(SignInResult result) async { // #enddocregion handle-signin, handle-confirm-signin-sms, handle-confirm-signin-new-password, handle-confirm-signin-custom-challenge, handle-confirm-signin-reset-password, handle-confirm-signin-confirm-signup, handle-confirm-signin-done, handle-confirm-signin-mfa-selection, handle-confirm-signin-totp-setup, handle-confirm-signin-totp-code, handle-confirm-signin-email-code, handle-confirm-signin-mfa-setup-selection, handle-confirm-signin-email-setup // #docregion handle-confirm-signin-mfa-selection case AuthSignInStep.continueSignInWithMfaSelection: - final allowedMfaTypes = result.nextStep.allowedMfaTypes; + final allowedMfaTypes = result.nextStep.allowedMfaTypes!; final selection = await _promptUserPreference(allowedMfaTypes); return _handleMfaSelection(selection); // #enddocregion handle-confirm-signin-mfa-selection @@ -126,7 +126,7 @@ Future _handleSignInResult(SignInResult result) async { // #enddocregion handle-confirm-signin-mfa-setup-selection // #docregion handle-confirm-signin-totp-setup case AuthSignInStep.continueSignInWithTotpSetup: - final totpSetupDetails = result.nextStep.totpSetupDetails; + final totpSetupDetails = result.nextStep.totpSetupDetails!; final setupUri = totpSetupDetails.getSetupUri(appName: 'MyApp'); safePrint('Open URI to complete setup: $setupUri'); // #enddocregion handle-confirm-signin-totp-setup @@ -276,7 +276,7 @@ Future signOutGlobally() async { if (result is CognitoCompleteSignOut) { safePrint('Sign out completed successfully'); } else if (result is CognitoPartialSignOut) { - final globalSignOutException = result.globalSignOutException; + final globalSignOutException = result.globalSignOutException!; final accessToken = globalSignOutException.accessToken; // Retry the global sign out using the access token, if desired // ... From dc03882f12ad3fb68fec2cb7983779c376e1571e Mon Sep 17 00:00:00 2001 From: ekjotmultani Date: Wed, 18 Sep 2024 13:40:33 -0700 Subject: [PATCH 07/25] added trailing commas to pass ci test --- .../lib/src/config/amplify_outputs/storage/storage_outputs.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart index 87f34ad570..10eac4b055 100644 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart @@ -13,7 +13,7 @@ part 'storage_outputs.g.dart'; class StorageOutputs with AWSEquatable, AWSSerializable, AWSDebuggable { /// {@macro amplify_core.amplify_outputs.storage_outputs} - const StorageOutputs({required this.awsRegion, required this.bucketName, this.buckets}); + const StorageOutputs({required this.awsRegion, required this.bucketName, this.buckets,}); factory StorageOutputs.fromJson(Map json) => From 6a3ea1db386bba9a272553a7fe552f81c7578f77 Mon Sep 17 00:00:00 2001 From: ekjotmultani <43255916+ekjotmultani@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:30:21 -0700 Subject: [PATCH 08/25] Delete devtools_options.yaml --- devtools_options.yaml | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 devtools_options.yaml diff --git a/devtools_options.yaml b/devtools_options.yaml deleted file mode 100644 index fa0b357c4f..0000000000 --- a/devtools_options.yaml +++ /dev/null @@ -1,3 +0,0 @@ -description: This file stores settings for Dart & Flutter DevTools. -documentation: https://docs.flutter.dev/tools/devtools/extensions#configure-extension-enablement-states -extensions: From f6b8cb3e8b13a6da89978d5dcb82c4429a36ed2e Mon Sep 17 00:00:00 2001 From: ekjotmultani Date: Wed, 18 Sep 2024 14:25:35 -0700 Subject: [PATCH 09/25] ran dart format on two failing files --- .../amplify_outputs/storage/bucket_outputs.dart | 14 ++++++++++---- .../amplify_outputs/storage/storage_outputs.dart | 7 +++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_outputs.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_outputs.dart index c3e05b87ec..e156f08567 100644 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_outputs.dart +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/bucket_outputs.dart @@ -9,18 +9,24 @@ part 'bucket_outputs.g.dart'; /// The Amplify Gen 2 outputs for Buckets in the Storage category. /// {@endtemplate} @zAmplifyOutputsSerializable -class BucketOutputs - with AWSEquatable, AWSSerializable, AWSDebuggable{ +class BucketOutputs + with AWSEquatable, AWSSerializable, AWSDebuggable { /// {@macro amplify_core.amplify_outputs.bucket_outputs} - const BucketOutputs({required this.name, required this.bucketName, required this.awsRegion,}); + const BucketOutputs({ + required this.name, + required this.bucketName, + required this.awsRegion, + }); factory BucketOutputs.fromJson(Map json) => _$BucketOutputsFromJson(json); /// The user friendly name of the bucket final String name; + /// The Amazon S3 bucket name. final String bucketName; + /// The AWS region of Amazon S3 resources. final String awsRegion; @@ -33,7 +39,7 @@ class BucketOutputs @override String get runtimeTypeName => 'BucketOutputs'; - + @override Object? toJson() { return _$BucketOutputsToJson(this); diff --git a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart index 10eac4b055..fec0ec0662 100644 --- a/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart +++ b/packages/amplify_core/lib/src/config/amplify_outputs/storage/storage_outputs.dart @@ -13,8 +13,11 @@ part 'storage_outputs.g.dart'; class StorageOutputs with AWSEquatable, AWSSerializable, AWSDebuggable { /// {@macro amplify_core.amplify_outputs.storage_outputs} - const StorageOutputs({required this.awsRegion, required this.bucketName, this.buckets,}); - + const StorageOutputs({ + required this.awsRegion, + required this.bucketName, + this.buckets, + }); factory StorageOutputs.fromJson(Map json) => _$StorageOutputsFromJson(json); From 9246fffe107c6e0e810e4ab7192da8c0ee8011ce Mon Sep 17 00:00:00 2001 From: ekjotmultani Date: Tue, 24 Sep 2024 11:01:33 -0700 Subject: [PATCH 10/25] updated resource.ts and backend.ts for multiple buckets in our infra-gen2 stack --- .../storage/main/amplify/storage/resource.ts | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/infra-gen2/backends/storage/main/amplify/storage/resource.ts b/infra-gen2/backends/storage/main/amplify/storage/resource.ts index 3fb921c12b..4bb947fa19 100644 --- a/infra-gen2/backends/storage/main/amplify/storage/resource.ts +++ b/infra-gen2/backends/storage/main/amplify/storage/resource.ts @@ -1,7 +1,25 @@ import { defineStorage } from "@aws-amplify/backend"; -export const storage = defineStorage({ - name: "Storage Integ Test main", +export const firstBucket = defineStorage({ + name: "Storage Integ Test main bucket", + isDefault: true, + access: (allow) => ({ + "public/*": [ + allow.guest.to(["read", "write", "delete"]), + allow.authenticated.to(["read", "delete", "write"]), + ], + "protected/{entity_id}/*": [ + allow.authenticated.to(["read"]), + allow.entity("identity").to(["read", "write", "delete"]), + ], + "private/{entity_id}/*": [ + allow.entity("identity").to(["read", "write", "delete"]), + ], + }), +}); + +export const secondBucket = defineStorage({ + name: "Storage Integ Test secondary bucket", access: (allow) => ({ "public/*": [ allow.guest.to(["read", "write", "delete"]), From 600e72580ec8a332bcdc8fdf315bf464d8401892 Mon Sep 17 00:00:00 2001 From: ekjotmultani Date: Tue, 24 Sep 2024 11:02:33 -0700 Subject: [PATCH 11/25] updated resource.ts and backend.ts for multiple buckets in our infra-gen2 stack --- .../backends/storage/main/amplify/backend.ts | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/infra-gen2/backends/storage/main/amplify/backend.ts b/infra-gen2/backends/storage/main/amplify/backend.ts index e9d462a0b3..c75ada8bba 100644 --- a/infra-gen2/backends/storage/main/amplify/backend.ts +++ b/infra-gen2/backends/storage/main/amplify/backend.ts @@ -1,26 +1,52 @@ import { defineBackend } from "@aws-amplify/backend"; import * as s3 from "aws-cdk-lib/aws-s3"; import { auth } from "./auth/resource"; -import { storage } from "./storage/resource"; +import { firstBucket, secondBucket } from "./storage/resource"; /** * @see https://docs.amplify.aws/react/build-a-backend/ to add storage, functions, and more */ const backend = defineBackend({ auth, - storage, + firstBucket, + secondBucket, }); // custom storage configurations -const s3Bucket = backend.storage.resources.bucket; +const s3Bucket = backend.firstBucket.resources.bucket; const cfnBucket = s3Bucket.node.defaultChild as s3.CfnBucket; +const s3SecondaryBucket = backend.secondBucket.resources.bucket; +const cfnSecondaryBucket = s3SecondaryBucket.node.defaultChild as s3.CfnBucket; cfnBucket.accelerateConfiguration = { accelerationStatus: "Enabled", }; +cfnSecondaryBucket.accelerateConfiguration = { + accelerationStatus: "Enabled", +}; + +// required to add the metadata header, which amplify-backend does not support +backend.firstBucket.resources.cfnResources.cfnBucket.corsConfiguration = { + corsRules: [ + { + allowedHeaders: ["*"], + allowedMethods: ["GET", "HEAD", "PUT", "POST", "DELETE"], + allowedOrigins: ["*"], + exposedHeaders: [ + "x-amz-server-side-encryption", + "x-amz-request-id", + "x-amz-id-2", + "ETag", + "x-amz-meta-description", + ], + maxAge: 3000, + }, + ], +}; + // required to add the metadata header, which amplify-backend does not support -backend.storage.resources.cfnResources.cfnBucket.corsConfiguration = { +backend.secondBucket.resources.cfnResources.cfnBucket.corsConfiguration = { corsRules: [ { allowedHeaders: ["*"], From 27532028504b6498c300e6f81e3e363539474863 Mon Sep 17 00:00:00 2001 From: NikaHsn Date: Wed, 25 Sep 2024 11:57:31 -0700 Subject: [PATCH 12/25] feat(storage): update s3 storage service to support multiple s3 client for multi-bucket support (#5493) --- .../storage/storage_bucket_from_outputs.dart | 27 +++++-- .../service/s3_client_info.dart | 12 +++ .../service/storage_s3_service_impl.dart | 79 +++++++++++++++---- 3 files changed, 99 insertions(+), 19 deletions(-) create mode 100644 packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/s3_client_info.dart diff --git a/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart b/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart index 4ee6cedb83..21ab788904 100644 --- a/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart +++ b/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart @@ -17,15 +17,32 @@ class StorageBucketFromOutputs implements StorageBucket { assert( storageOutputs != null, const InvalidStorageBucketException( - 'Amplify Storage is not configured.', + 'Amplify Outputs file does not have storage configuration.', recoverySuggestion: - 'Make sure storage exists in the Amplify Outputs file.', + 'Make sure Amplify Storage is configured and the Amplify Outputs ' + 'file has storage configuration.', + ), + ); + final buckets = storageOutputs!.buckets; + if (buckets == null) { + throw const InvalidStorageBucketException( + 'Amplify Outputs storage configuration does not have buckets specified.', + recoverySuggestion: + 'Make sure Amplify Outputs file has storage configuration with ' + 'buckets specified.', + ); + } + final bucket = buckets.singleWhere( + (e) => e.name == _name, + orElse: () => throw const InvalidStorageBucketException( + 'Unable to lookup bucket from provided name in Amplify Outputs file.', + recoverySuggestion: 'Make sure Amplify Outputs file has the specified ' + 'bucket configuration.', ), ); - // TODO(nikahsn): fix after adding buckets to StorageOutputs. return BucketInfo( - bucketName: _name, - region: storageOutputs!.awsRegion, + bucketName: bucket.bucketName, + region: bucket.awsRegion, ); } } diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/s3_client_info.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/s3_client_info.dart new file mode 100644 index 0000000000..162c140dcd --- /dev/null +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/s3_client_info.dart @@ -0,0 +1,12 @@ +import 'package:amplify_storage_s3_dart/src/sdk/src/s3/s3_client.dart'; +import 'package:meta/meta.dart'; +import 'package:smithy_aws/smithy_aws.dart'; + +/// It holds Amazon S3 client information. +@internal +class S3ClientInfo { + const S3ClientInfo({required this.client, required this.config}); + + final S3Client client; + final S3ClientConfig config; +} diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index f749de67f3..33f32dba3b 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -14,6 +14,7 @@ import 'package:amplify_storage_s3_dart/src/path_resolver/s3_path_resolver.dart' import 'package:amplify_storage_s3_dart/src/sdk/s3.dart' as s3; import 'package:amplify_storage_s3_dart/src/sdk/src/s3/common/endpoint_resolver.dart' as endpoint_resolver; +import 'package:amplify_storage_s3_dart/src/storage_s3_service/service/s3_client_info.dart'; import 'package:amplify_storage_s3_dart/src/storage_s3_service/storage_s3_service.dart'; import 'package:amplify_storage_s3_dart/src/storage_s3_service/transfer/transfer.dart' as transfer; @@ -84,10 +85,8 @@ class StorageS3Service { ..supportedProtocols = SupportedProtocols.http1, ), _pathResolver = pathResolver, + _credentialsProvider = credentialsProvider, _logger = logger, - // dependencyManager.get() => sigv4.AWSSigV4Signer is used for unit tests - _awsSigV4Signer = dependencyManager.get() ?? - sigv4.AWSSigV4Signer(credentialsProvider: credentialsProvider), _dependencyManager = dependencyManager, _serviceStartingTime = DateTime.now(); @@ -101,14 +100,10 @@ class StorageS3Service { final s3.S3Client _defaultS3Client; final S3PathResolver _pathResolver; final AWSLogger _logger; - final sigv4.AWSSigV4Signer _awsSigV4Signer; final DependencyManager _dependencyManager; final DateTime _serviceStartingTime; - - sigv4.AWSCredentialScope get _signerScope => sigv4.AWSCredentialScope( - region: _storageOutputs.awsRegion, - service: AWSService.s3, - ); + final AWSIamAmplifyAuthProvider _credentialsProvider; + final Map _s3ClientsInfo = {}; transfer.TransferDatabase get _transferDatabase => _dependencyManager.getOrCreate(); @@ -261,10 +256,20 @@ class StorageS3Service { path: '/$resolvedPath', ); + // dependencyManager.get() is used for unit tests + final awsSigV4Signer = _dependencyManager.get() ?? + sigv4.AWSSigV4Signer( + credentialsProvider: _credentialsProvider, + ); + final signerScope = sigv4.AWSCredentialScope( + region: _storageOutputs.awsRegion, + service: AWSService.s3, + ); + return S3GetUrlResult( - url: await _awsSigV4Signer.presign( + url: await awsSigV4Signer.presign( urlRequest, - credentialScope: _signerScope, + credentialScope: signerScope, expiresIn: s3PluginOptions.expiresIn, serviceConfiguration: _defaultS3SignerConfiguration, ), @@ -323,12 +328,17 @@ class StorageS3Service { void Function(S3TransferProgress)? onProgress, FutureOr Function()? onDone, FutureOr Function()? onError, + StorageBucket? bucket, }) { + // ignore: invalid_use_of_internal_member + final bucketName = bucket?.resolveBucketInfo(_storageOutputs).bucketName ?? + _storageOutputs.bucketName; + final s3ClientInfo = _getS3ClientInfo(bucket); final uploadDataTask = S3UploadTask.fromDataPayload( dataPayload, - s3Client: _defaultS3Client, - defaultS3ClientConfig: _defaultS3ClientConfig, - bucket: _storageOutputs.bucketName, + s3Client: s3ClientInfo.client, + defaultS3ClientConfig: s3ClientInfo.config, + bucket: bucketName, path: path, options: options, pathResolver: _pathResolver, @@ -611,4 +621,45 @@ class StorageS3Service { } } } + + S3ClientInfo _getS3ClientInfo(StorageBucket? storageBucket) { + if (storageBucket == null) { + return S3ClientInfo( + client: _defaultS3Client, + config: _defaultS3ClientConfig, + ); + } + // ignore: invalid_use_of_internal_member + final bucketInfo = storageBucket.resolveBucketInfo(_storageOutputs); + if (_s3ClientsInfo[bucketInfo.bucketName] != null) { + return _s3ClientsInfo[bucketInfo.bucketName]!; + } + + final usePathStyle = bucketInfo.bucketName.contains('.'); + if (usePathStyle) { + _logger.warn( + 'Since your bucket name contains dots (`"."`), the StorageS3 plugin' + ' will use path style URLs to communicate with the S3 service. S3' + ' Transfer acceleration is not supported for path style URLs. For more' + ' information, refer to:' + ' https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html'); + } + final s3ClientConfig = smithy_aws.S3ClientConfig( + signerConfiguration: _defaultS3SignerConfiguration, + usePathStyle: usePathStyle, + ); + final s3Client = s3.S3Client( + region: bucketInfo.region, + credentialsProvider: _credentialsProvider, + s3ClientConfig: s3ClientConfig, + client: AmplifyHttpClient(_dependencyManager) + ..supportedProtocols = SupportedProtocols.http1, + ); + final s3ClientInfo = S3ClientInfo( + client: s3Client, + config: s3ClientConfig, + ); + _s3ClientsInfo[bucketInfo.bucketName] = s3ClientInfo; + return s3ClientInfo; + } } From 0658d24a925f8a5604f3153ca122aee3f30de7a4 Mon Sep 17 00:00:00 2001 From: NikaHsn Date: Mon, 14 Oct 2024 12:07:34 -0700 Subject: [PATCH 13/25] feat(storage): update uploadData API to accept optional storage bucket param (#5540) --- .../category/amplify_storage_category.dart | 1 + .../amplify_storage_plugin_interface.dart | 1 + .../lib/src/types/storage/bucket_info.dart | 10 +- .../storage/storage_bucket_from_outputs.dart | 7 +- .../types/storage/storage_bucket_test.dart | 87 ++++++++++++ .../lib/src/amplify_storage_s3_dart_impl.dart | 2 + .../service/s3_client_info.dart | 9 +- .../service/storage_s3_service_impl.dart | 36 +++-- .../service/task/s3_upload_task.dart | 29 ++-- .../transfer/database/database_io.dart | 24 +++- .../transfer/database/tables.dart | 6 + .../transfer/database/tables.drift.dart | 133 ++++++++++++++++-- .../transfer/database/transfer_record.dart | 8 ++ .../transfer/database/transfer_record.g.dart | 4 + .../test/amplify_storage_s3_dart_test.dart | 50 +++++++ .../storage_s3_service_test.dart | 35 ++++- .../task/s3_upload_task_test.dart | 86 +++++++---- .../transfer/database_html_test.dart | 4 + .../test/test_utils/mocks.dart | 4 + 19 files changed, 466 insertions(+), 70 deletions(-) create mode 100644 packages/amplify_core/test/types/storage/storage_bucket_test.dart diff --git a/packages/amplify_core/lib/src/category/amplify_storage_category.dart b/packages/amplify_core/lib/src/category/amplify_storage_category.dart index 416446d12e..c3f4cfbe2f 100644 --- a/packages/amplify_core/lib/src/category/amplify_storage_category.dart +++ b/packages/amplify_core/lib/src/category/amplify_storage_category.dart @@ -144,6 +144,7 @@ class StorageCategory extends AmplifyCategory { required StoragePath path, void Function(StorageTransferProgress)? onProgress, StorageUploadDataOptions? options, + StorageBucket? bucket, }) { return identifyCall( StorageCategoryMethod.uploadData, diff --git a/packages/amplify_core/lib/src/plugin/amplify_storage_plugin_interface.dart b/packages/amplify_core/lib/src/plugin/amplify_storage_plugin_interface.dart index c4d874d8a0..efb891fbef 100644 --- a/packages/amplify_core/lib/src/plugin/amplify_storage_plugin_interface.dart +++ b/packages/amplify_core/lib/src/plugin/amplify_storage_plugin_interface.dart @@ -63,6 +63,7 @@ abstract class StoragePluginInterface extends AmplifyPluginInterface { required StorageDataPayload data, void Function(StorageTransferProgress)? onProgress, StorageUploadDataOptions? options, + StorageBucket? bucket, }) { throw UnimplementedError('uploadData() has not been implemented.'); } diff --git a/packages/amplify_core/lib/src/types/storage/bucket_info.dart b/packages/amplify_core/lib/src/types/storage/bucket_info.dart index a3c065ef23..a30811b333 100644 --- a/packages/amplify_core/lib/src/types/storage/bucket_info.dart +++ b/packages/amplify_core/lib/src/types/storage/bucket_info.dart @@ -1,9 +1,17 @@ +import 'package:amplify_core/amplify_core.dart'; + /// {@template amplify_core.storage.bucket_info} /// Presents a storage bucket information. /// {@endtemplate} -class BucketInfo { +class BucketInfo with AWSEquatable { /// {@macro amplify_core.storage.bucket_info} const BucketInfo({required this.bucketName, required this.region}); final String bucketName; final String region; + + @override + List get props => [ + bucketName, + region, + ]; } diff --git a/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart b/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart index 21ab788904..ed22c24aa2 100644 --- a/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart +++ b/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart @@ -16,12 +16,7 @@ class StorageBucketFromOutputs implements StorageBucket { BucketInfo resolveBucketInfo(StorageOutputs? storageOutputs) { assert( storageOutputs != null, - const InvalidStorageBucketException( - 'Amplify Outputs file does not have storage configuration.', - recoverySuggestion: - 'Make sure Amplify Storage is configured and the Amplify Outputs ' - 'file has storage configuration.', - ), + 'storageOutputs can not be null', ); final buckets = storageOutputs!.buckets; if (buckets == null) { diff --git a/packages/amplify_core/test/types/storage/storage_bucket_test.dart b/packages/amplify_core/test/types/storage/storage_bucket_test.dart new file mode 100644 index 0000000000..edae2ab4af --- /dev/null +++ b/packages/amplify_core/test/types/storage/storage_bucket_test.dart @@ -0,0 +1,87 @@ +import 'package:amplify_core/amplify_core.dart'; +import 'package:amplify_core/src/config/amplify_outputs/storage/bucket_outputs.dart'; +import 'package:amplify_core/src/config/amplify_outputs/storage/storage_outputs.dart'; +import 'package:test/test.dart'; + +void main() { + group('Storage bucket resolve BucketInfo', () { + const defaultBucketOutputs = BucketOutputs( + name: 'default-bucket-friendly-name', + bucketName: 'default-bucket-unique-name', + awsRegion: 'default-bucket-aws-region', + ); + const secondBucketOutputs = BucketOutputs( + name: 'second-bucket-friendly-name', + bucketName: 'second-bucket-unique-name', + awsRegion: 'second-bucket-aws-region', + ); + final defaultBucketInfo = BucketInfo( + bucketName: defaultBucketOutputs.bucketName, + region: defaultBucketOutputs.awsRegion, + ); + final secondBucketInfo = BucketInfo( + bucketName: secondBucketOutputs.bucketName, + region: secondBucketOutputs.awsRegion, + ); + final testStorageOutputsMultiBucket = StorageOutputs( + awsRegion: defaultBucketOutputs.awsRegion, + bucketName: defaultBucketOutputs.bucketName, + buckets: [ + defaultBucketOutputs, + secondBucketOutputs, + ], + ); + final testStorageOutputsSingleBucket = StorageOutputs( + awsRegion: defaultBucketOutputs.awsRegion, + bucketName: defaultBucketOutputs.bucketName, + ); + + test( + 'should return same bucket info when storage bucket is created from' + ' a bucket info', () { + final storageBucket = StorageBucket.fromBucketInfo( + defaultBucketInfo, + ); + final bucketInfo = storageBucket.resolveBucketInfo(null); + expect(bucketInfo, defaultBucketInfo); + }); + + test( + 'should return bucket info when storage bucket is created from' + ' buckets in storage outputs', () { + final storageBucket = StorageBucket.fromOutputs(secondBucketOutputs.name); + final bucketInfo = + storageBucket.resolveBucketInfo(testStorageOutputsMultiBucket); + expect(bucketInfo, secondBucketInfo); + }); + + test( + 'should throw assertion error when storage bucket is created from' + ' outputs and storage outputs is null', () { + final storageBucket = + StorageBucket.fromOutputs(defaultBucketOutputs.name); + expect( + () => storageBucket.resolveBucketInfo(null), + throwsA(isA()), + ); + }); + test( + 'should throw exception when storage bucket is created from outputs and' + ' storage outputs does not have buckets', () { + final storageBucket = StorageBucket.fromOutputs('bucket-name'); + expect( + () => storageBucket.resolveBucketInfo(testStorageOutputsSingleBucket), + throwsA(isA()), + ); + }); + test( + 'should throw exception when storage bucket is created from outputs and' + ' bucket name does not match any bucket in storage outputs', () { + final storageBucket = StorageBucket.fromOutputs('invalid-bucket-name'); + expect( + () => storageBucket.resolveBucketInfo(testStorageOutputsMultiBucket), + throwsA(isA()), + ); + }); + }); +} diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index cc709c5802..f4ec4035c4 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -274,6 +274,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface required StoragePath path, void Function(S3TransferProgress)? onProgress, StorageUploadDataOptions? options, + StorageBucket? bucket, }) { final s3PluginOptions = reifyPluginOptions( pluginOptions: options?.pluginOptions, @@ -290,6 +291,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface dataPayload: data, options: s3Options, onProgress: onProgress, + bucket: bucket, ); return S3UploadDataOperation( diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/s3_client_info.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/s3_client_info.dart index 162c140dcd..d70adc524d 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/s3_client_info.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/s3_client_info.dart @@ -5,8 +5,15 @@ import 'package:smithy_aws/smithy_aws.dart'; /// It holds Amazon S3 client information. @internal class S3ClientInfo { - const S3ClientInfo({required this.client, required this.config}); + const S3ClientInfo({ + required this.client, + required this.config, + required this.bucketName, + required this.awsRegion, + }); final S3Client client; final S3ClientConfig config; + final String bucketName; + final String awsRegion; } diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index 33f32dba3b..86a355eac9 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -330,15 +330,13 @@ class StorageS3Service { FutureOr Function()? onError, StorageBucket? bucket, }) { - // ignore: invalid_use_of_internal_member - final bucketName = bucket?.resolveBucketInfo(_storageOutputs).bucketName ?? - _storageOutputs.bucketName; - final s3ClientInfo = _getS3ClientInfo(bucket); + final s3ClientInfo = getS3ClientInfo(storageBucket: bucket); final uploadDataTask = S3UploadTask.fromDataPayload( dataPayload, s3Client: s3ClientInfo.client, - defaultS3ClientConfig: s3ClientInfo.config, - bucket: bucketName, + s3ClientConfig: s3ClientInfo.config, + bucket: s3ClientInfo.bucketName, + awsRegion: s3ClientInfo.awsRegion, path: path, options: options, pathResolver: _pathResolver, @@ -376,8 +374,9 @@ class StorageS3Service { final uploadDataTask = S3UploadTask.fromAWSFile( localFile, s3Client: _defaultS3Client, - defaultS3ClientConfig: _defaultS3ClientConfig, + s3ClientConfig: _defaultS3ClientConfig, bucket: _storageOutputs.bucketName, + awsRegion: _storageOutputs.awsRegion, path: path, options: uploadDataOptions, pathResolver: _pathResolver, @@ -604,17 +603,23 @@ class StorageS3Service { Future abortIncompleteMultipartUploads() async { final records = await _transferDatabase .getMultipartUploadRecordsCreatedBefore(_serviceStartingTime); - for (final record in records) { + final bucketInfo = BucketInfo( + bucketName: record.bucketName ?? _storageOutputs.bucketName, + region: record.awsRegion ?? _storageOutputs.awsRegion, + ); final request = s3.AbortMultipartUploadRequest.build((builder) { builder - ..bucket = _storageOutputs.bucketName + ..bucket = bucketInfo.bucketName ..key = record.objectKey ..uploadId = record.uploadId; }); + final s3Client = getS3ClientInfo( + storageBucket: StorageBucket.fromBucketInfo(bucketInfo), + ).client; try { - await _defaultS3Client.abortMultipartUpload(request).result; + await s3Client.abortMultipartUpload(request).result; await _transferDatabase.deleteTransferRecords(record.uploadId); } on Exception catch (error) { _logger.error('Failed to abort multipart upload due to: $error'); @@ -622,11 +627,18 @@ class StorageS3Service { } } - S3ClientInfo _getS3ClientInfo(StorageBucket? storageBucket) { + /// Creates and caches [S3ClientInfo] given the optional [storageBucket] + /// parameter. If the optional parameter is not provided it uses + /// StorageOutputs default bucket to create the [S3ClientInfo]. + @internal + @visibleForTesting + S3ClientInfo getS3ClientInfo({StorageBucket? storageBucket}) { if (storageBucket == null) { return S3ClientInfo( client: _defaultS3Client, config: _defaultS3ClientConfig, + bucketName: _storageOutputs.bucketName, + awsRegion: _storageOutputs.awsRegion, ); } // ignore: invalid_use_of_internal_member @@ -658,6 +670,8 @@ class StorageS3Service { final s3ClientInfo = S3ClientInfo( client: s3Client, config: s3ClientConfig, + bucketName: bucketInfo.bucketName, + awsRegion: bucketInfo.region, ); _s3ClientsInfo[bucketInfo.bucketName] = s3ClientInfo; return s3ClientInfo; diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/task/s3_upload_task.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/task/s3_upload_task.dart index dbecca6d44..364c74237e 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/task/s3_upload_task.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/task/s3_upload_task.dart @@ -48,9 +48,10 @@ const fallbackContentType = 'application/octet-stream'; class S3UploadTask { S3UploadTask._({ required s3.S3Client s3Client, - required smithy_aws.S3ClientConfig defaultS3ClientConfig, + required smithy_aws.S3ClientConfig s3ClientConfig, required S3PathResolver pathResolver, required String bucket, + required String awsRegion, required StoragePath path, required StorageUploadDataOptions options, S3DataPayload? dataPayload, @@ -59,9 +60,10 @@ class S3UploadTask { required AWSLogger logger, required transfer.TransferDatabase transferDatabase, }) : _s3Client = s3Client, - _defaultS3ClientConfig = defaultS3ClientConfig, + _s3ClientConfig = s3ClientConfig, _pathResolver = pathResolver, _bucket = bucket, + _awsRegion = awsRegion, _path = path, _options = options, _dataPayload = dataPayload, @@ -81,9 +83,10 @@ class S3UploadTask { S3UploadTask.fromDataPayload( S3DataPayload dataPayload, { required s3.S3Client s3Client, - required smithy_aws.S3ClientConfig defaultS3ClientConfig, + required smithy_aws.S3ClientConfig s3ClientConfig, required S3PathResolver pathResolver, required String bucket, + required String awsRegion, required StoragePath path, required StorageUploadDataOptions options, void Function(S3TransferProgress)? onProgress, @@ -91,9 +94,10 @@ class S3UploadTask { required transfer.TransferDatabase transferDatabase, }) : this._( s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: s3ClientConfig, pathResolver: pathResolver, bucket: bucket, + awsRegion: awsRegion, path: path, dataPayload: dataPayload, options: options, @@ -108,9 +112,10 @@ class S3UploadTask { S3UploadTask.fromAWSFile( AWSFile localFile, { required s3.S3Client s3Client, - required smithy_aws.S3ClientConfig defaultS3ClientConfig, + required smithy_aws.S3ClientConfig s3ClientConfig, required S3PathResolver pathResolver, required String bucket, + required String awsRegion, required StoragePath path, required StorageUploadDataOptions options, void Function(S3TransferProgress)? onProgress, @@ -118,9 +123,10 @@ class S3UploadTask { required transfer.TransferDatabase transferDatabase, }) : this._( s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: s3ClientConfig, pathResolver: pathResolver, bucket: bucket, + awsRegion: awsRegion, path: path, localFile: localFile, options: options, @@ -135,9 +141,10 @@ class S3UploadTask { final Completer _uploadCompleter = Completer(); final s3.S3Client _s3Client; - final smithy_aws.S3ClientConfig _defaultS3ClientConfig; + final smithy_aws.S3ClientConfig _s3ClientConfig; final S3PathResolver _pathResolver; final String _bucket; + final String _awsRegion; final StoragePath _path; final StorageUploadDataOptions _options; final void Function(S3TransferProgress)? _onProgress; @@ -191,7 +198,7 @@ class S3UploadTask { /// Should be used only internally. Future start() async { if (_s3PluginOptions.useAccelerateEndpoint && - _defaultS3ClientConfig.usePathStyle) { + _s3ClientConfig.usePathStyle) { _completeUploadWithError(s3_exception.accelerateEndpointUnusable); return; } @@ -328,7 +335,7 @@ class S3UploadTask { try { _putObjectOperation = _s3Client.putObject( putObjectRequest, - s3ClientConfig: _defaultS3ClientConfig.copyWith( + s3ClientConfig: _s3ClientConfig.copyWith( useAcceleration: _s3PluginOptions.useAccelerateEndpoint, ), ); @@ -497,6 +504,8 @@ class S3UploadTask { TransferRecord( uploadId: uploadId, objectKey: _resolvedPath, + bucketName: _bucket, + awsRegion: _awsRegion, createdAt: DateTime.now(), ), ); @@ -655,7 +664,7 @@ class S3UploadTask { try { final operation = _s3Client.uploadPart( request, - s3ClientConfig: _defaultS3ClientConfig.copyWith( + s3ClientConfig: _s3ClientConfig.copyWith( useAcceleration: _s3PluginOptions.useAccelerateEndpoint, ), ); diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/database_io.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/database_io.dart index a16a66bcf3..1c105d8b32 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/database_io.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/database_io.dart @@ -35,7 +35,25 @@ class TransferDatabase extends $TransferDatabase // Bump the version number when any alteration is made into tables.dart @override - int get schemaVersion => 1; + int get schemaVersion => 2; + + @override + MigrationStrategy get migration { + return MigrationStrategy( + onCreate: (Migrator m) async { + await m.createAll(); + }, + onUpgrade: (Migrator m, int from, int to) async { + // Note: From schemaVersion 1->2 we added bucketName and awsRegion. + // they are nullable columns so that on upgrade we need to update + // the transferRecords table to add these two columns + if (from < 2) { + await m.addColumn(transferRecords, transferRecords.bucketName); + await m.addColumn(transferRecords, transferRecords.awsRegion); + } + }, + ); + } @override Future> getMultipartUploadRecordsCreatedBefore( @@ -52,6 +70,8 @@ class TransferDatabase extends $TransferDatabase objectKey: e.objectKey, uploadId: e.uploadId, createdAt: DateTime.parse(e.createdAt), + bucketName: e.bucketName, + awsRegion: e.awsRegion, ), ) .get(); @@ -63,6 +83,8 @@ class TransferDatabase extends $TransferDatabase uploadId: record.uploadId, objectKey: record.objectKey, createdAt: record.createdAt.toIso8601String(), + bucketName: Value(record.bucketName), + awsRegion: Value(record.awsRegion), ); final value = await into(transferRecords).insert(entry); return value.toString(); diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/tables.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/tables.dart index 00d5b5b22d..4d13dcea2a 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/tables.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/tables.dart @@ -20,4 +20,10 @@ class TransferRecords extends Table { /// Timestamp of [uploadId] creation. TextColumn get createdAt => text()(); + + /// Amazon S3 bucket name. + TextColumn get bucketName => text().nullable()(); + + /// AWS region of Amazon S3 bucket. + TextColumn get awsRegion => text().nullable()(); } diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/tables.drift.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/tables.drift.dart index 30ebd91612..f50e5c57ce 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/tables.drift.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/tables.drift.dart @@ -38,8 +38,21 @@ class $TransferRecordsTable extends i2.TransferRecords late final i0.GeneratedColumn createdAt = i0.GeneratedColumn( 'created_at', aliasedName, false, type: i0.DriftSqlType.string, requiredDuringInsert: true); + static const i0.VerificationMeta _bucketNameMeta = + const i0.VerificationMeta('bucketName'); @override - List get $columns => [id, uploadId, objectKey, createdAt]; + late final i0.GeneratedColumn bucketName = i0.GeneratedColumn( + 'bucket_name', aliasedName, true, + type: i0.DriftSqlType.string, requiredDuringInsert: false); + static const i0.VerificationMeta _awsRegionMeta = + const i0.VerificationMeta('awsRegion'); + @override + late final i0.GeneratedColumn awsRegion = i0.GeneratedColumn( + 'aws_region', aliasedName, true, + type: i0.DriftSqlType.string, requiredDuringInsert: false); + @override + List get $columns => + [id, uploadId, objectKey, createdAt, bucketName, awsRegion]; @override String get aliasedName => _alias ?? actualTableName; @override @@ -72,6 +85,16 @@ class $TransferRecordsTable extends i2.TransferRecords } else if (isInserting) { context.missing(_createdAtMeta); } + if (data.containsKey('bucket_name')) { + context.handle( + _bucketNameMeta, + bucketName.isAcceptableOrUnknown( + data['bucket_name']!, _bucketNameMeta)); + } + if (data.containsKey('aws_region')) { + context.handle(_awsRegionMeta, + awsRegion.isAcceptableOrUnknown(data['aws_region']!, _awsRegionMeta)); + } return context; } @@ -89,6 +112,10 @@ class $TransferRecordsTable extends i2.TransferRecords .read(i0.DriftSqlType.string, data['${effectivePrefix}object_key'])!, createdAt: attachedDatabase.typeMapping .read(i0.DriftSqlType.string, data['${effectivePrefix}created_at'])!, + bucketName: attachedDatabase.typeMapping + .read(i0.DriftSqlType.string, data['${effectivePrefix}bucket_name']), + awsRegion: attachedDatabase.typeMapping + .read(i0.DriftSqlType.string, data['${effectivePrefix}aws_region']), ); } @@ -111,11 +138,19 @@ class TransferRecord extends i0.DataClass /// Timestamp of [uploadId] creation. final String createdAt; + + /// Amazon S3 bucket name. + final String? bucketName; + + /// AWS region of Amazon S3 bucket. + final String? awsRegion; const TransferRecord( {required this.id, required this.uploadId, required this.objectKey, - required this.createdAt}); + required this.createdAt, + this.bucketName, + this.awsRegion}); @override Map toColumns(bool nullToAbsent) { final map = {}; @@ -123,6 +158,12 @@ class TransferRecord extends i0.DataClass map['upload_id'] = i0.Variable(uploadId); map['object_key'] = i0.Variable(objectKey); map['created_at'] = i0.Variable(createdAt); + if (!nullToAbsent || bucketName != null) { + map['bucket_name'] = i0.Variable(bucketName); + } + if (!nullToAbsent || awsRegion != null) { + map['aws_region'] = i0.Variable(awsRegion); + } return map; } @@ -132,6 +173,12 @@ class TransferRecord extends i0.DataClass uploadId: i0.Value(uploadId), objectKey: i0.Value(objectKey), createdAt: i0.Value(createdAt), + bucketName: bucketName == null && nullToAbsent + ? const i0.Value.absent() + : i0.Value(bucketName), + awsRegion: awsRegion == null && nullToAbsent + ? const i0.Value.absent() + : i0.Value(awsRegion), ); } @@ -143,6 +190,8 @@ class TransferRecord extends i0.DataClass uploadId: serializer.fromJson(json['uploadId']), objectKey: serializer.fromJson(json['objectKey']), createdAt: serializer.fromJson(json['createdAt']), + bucketName: serializer.fromJson(json['bucketName']), + awsRegion: serializer.fromJson(json['awsRegion']), ); } @override @@ -153,16 +202,25 @@ class TransferRecord extends i0.DataClass 'uploadId': serializer.toJson(uploadId), 'objectKey': serializer.toJson(objectKey), 'createdAt': serializer.toJson(createdAt), + 'bucketName': serializer.toJson(bucketName), + 'awsRegion': serializer.toJson(awsRegion), }; } i1.TransferRecord copyWith( - {int? id, String? uploadId, String? objectKey, String? createdAt}) => + {int? id, + String? uploadId, + String? objectKey, + String? createdAt, + i0.Value bucketName = const i0.Value.absent(), + i0.Value awsRegion = const i0.Value.absent()}) => i1.TransferRecord( id: id ?? this.id, uploadId: uploadId ?? this.uploadId, objectKey: objectKey ?? this.objectKey, createdAt: createdAt ?? this.createdAt, + bucketName: bucketName.present ? bucketName.value : this.bucketName, + awsRegion: awsRegion.present ? awsRegion.value : this.awsRegion, ); @override String toString() { @@ -170,13 +228,16 @@ class TransferRecord extends i0.DataClass ..write('id: $id, ') ..write('uploadId: $uploadId, ') ..write('objectKey: $objectKey, ') - ..write('createdAt: $createdAt') + ..write('createdAt: $createdAt, ') + ..write('bucketName: $bucketName, ') + ..write('awsRegion: $awsRegion') ..write(')')) .toString(); } @override - int get hashCode => Object.hash(id, uploadId, objectKey, createdAt); + int get hashCode => + Object.hash(id, uploadId, objectKey, createdAt, bucketName, awsRegion); @override bool operator ==(Object other) => identical(this, other) || @@ -184,7 +245,9 @@ class TransferRecord extends i0.DataClass other.id == this.id && other.uploadId == this.uploadId && other.objectKey == this.objectKey && - other.createdAt == this.createdAt); + other.createdAt == this.createdAt && + other.bucketName == this.bucketName && + other.awsRegion == this.awsRegion); } class TransferRecordsCompanion extends i0.UpdateCompanion { @@ -192,17 +255,23 @@ class TransferRecordsCompanion extends i0.UpdateCompanion { final i0.Value uploadId; final i0.Value objectKey; final i0.Value createdAt; + final i0.Value bucketName; + final i0.Value awsRegion; const TransferRecordsCompanion({ this.id = const i0.Value.absent(), this.uploadId = const i0.Value.absent(), this.objectKey = const i0.Value.absent(), this.createdAt = const i0.Value.absent(), + this.bucketName = const i0.Value.absent(), + this.awsRegion = const i0.Value.absent(), }); TransferRecordsCompanion.insert({ this.id = const i0.Value.absent(), required String uploadId, required String objectKey, required String createdAt, + this.bucketName = const i0.Value.absent(), + this.awsRegion = const i0.Value.absent(), }) : uploadId = i0.Value(uploadId), objectKey = i0.Value(objectKey), createdAt = i0.Value(createdAt); @@ -211,12 +280,16 @@ class TransferRecordsCompanion extends i0.UpdateCompanion { i0.Expression? uploadId, i0.Expression? objectKey, i0.Expression? createdAt, + i0.Expression? bucketName, + i0.Expression? awsRegion, }) { return i0.RawValuesInsertable({ if (id != null) 'id': id, if (uploadId != null) 'upload_id': uploadId, if (objectKey != null) 'object_key': objectKey, if (createdAt != null) 'created_at': createdAt, + if (bucketName != null) 'bucket_name': bucketName, + if (awsRegion != null) 'aws_region': awsRegion, }); } @@ -224,12 +297,16 @@ class TransferRecordsCompanion extends i0.UpdateCompanion { {i0.Value? id, i0.Value? uploadId, i0.Value? objectKey, - i0.Value? createdAt}) { + i0.Value? createdAt, + i0.Value? bucketName, + i0.Value? awsRegion}) { return i1.TransferRecordsCompanion( id: id ?? this.id, uploadId: uploadId ?? this.uploadId, objectKey: objectKey ?? this.objectKey, createdAt: createdAt ?? this.createdAt, + bucketName: bucketName ?? this.bucketName, + awsRegion: awsRegion ?? this.awsRegion, ); } @@ -248,6 +325,12 @@ class TransferRecordsCompanion extends i0.UpdateCompanion { if (createdAt.present) { map['created_at'] = i0.Variable(createdAt.value); } + if (bucketName.present) { + map['bucket_name'] = i0.Variable(bucketName.value); + } + if (awsRegion.present) { + map['aws_region'] = i0.Variable(awsRegion.value); + } return map; } @@ -257,7 +340,9 @@ class TransferRecordsCompanion extends i0.UpdateCompanion { ..write('id: $id, ') ..write('uploadId: $uploadId, ') ..write('objectKey: $objectKey, ') - ..write('createdAt: $createdAt') + ..write('createdAt: $createdAt, ') + ..write('bucketName: $bucketName, ') + ..write('awsRegion: $awsRegion') ..write(')')) .toString(); } @@ -269,6 +354,8 @@ typedef $$TransferRecordsTableInsertCompanionBuilder required String uploadId, required String objectKey, required String createdAt, + i0.Value bucketName, + i0.Value awsRegion, }); typedef $$TransferRecordsTableUpdateCompanionBuilder = i1.TransferRecordsCompanion Function({ @@ -276,6 +363,8 @@ typedef $$TransferRecordsTableUpdateCompanionBuilder i0.Value uploadId, i0.Value objectKey, i0.Value createdAt, + i0.Value bucketName, + i0.Value awsRegion, }); class $$TransferRecordsTableTableManager extends i0.RootTableManager< @@ -303,24 +392,32 @@ class $$TransferRecordsTableTableManager extends i0.RootTableManager< i0.Value uploadId = const i0.Value.absent(), i0.Value objectKey = const i0.Value.absent(), i0.Value createdAt = const i0.Value.absent(), + i0.Value bucketName = const i0.Value.absent(), + i0.Value awsRegion = const i0.Value.absent(), }) => i1.TransferRecordsCompanion( id: id, uploadId: uploadId, objectKey: objectKey, createdAt: createdAt, + bucketName: bucketName, + awsRegion: awsRegion, ), getInsertCompanionBuilder: ({ i0.Value id = const i0.Value.absent(), required String uploadId, required String objectKey, required String createdAt, + i0.Value bucketName = const i0.Value.absent(), + i0.Value awsRegion = const i0.Value.absent(), }) => i1.TransferRecordsCompanion.insert( id: id, uploadId: uploadId, objectKey: objectKey, createdAt: createdAt, + bucketName: bucketName, + awsRegion: awsRegion, ), )); } @@ -360,6 +457,16 @@ class $$TransferRecordsTableFilterComposer column: $state.table.createdAt, builder: (column, joinBuilders) => i0.ColumnFilters(column, joinBuilders: joinBuilders)); + + i0.ColumnFilters get bucketName => $state.composableBuilder( + column: $state.table.bucketName, + builder: (column, joinBuilders) => + i0.ColumnFilters(column, joinBuilders: joinBuilders)); + + i0.ColumnFilters get awsRegion => $state.composableBuilder( + column: $state.table.awsRegion, + builder: (column, joinBuilders) => + i0.ColumnFilters(column, joinBuilders: joinBuilders)); } class $$TransferRecordsTableOrderingComposer extends i0 @@ -384,4 +491,14 @@ class $$TransferRecordsTableOrderingComposer extends i0 column: $state.table.createdAt, builder: (column, joinBuilders) => i0.ColumnOrderings(column, joinBuilders: joinBuilders)); + + i0.ColumnOrderings get bucketName => $state.composableBuilder( + column: $state.table.bucketName, + builder: (column, joinBuilders) => + i0.ColumnOrderings(column, joinBuilders: joinBuilders)); + + i0.ColumnOrderings get awsRegion => $state.composableBuilder( + column: $state.table.awsRegion, + builder: (column, joinBuilders) => + i0.ColumnOrderings(column, joinBuilders: joinBuilders)); } diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/transfer_record.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/transfer_record.dart index 3607e8a0ed..e5911c8d22 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/transfer_record.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/transfer_record.dart @@ -18,6 +18,8 @@ class TransferRecord { required this.uploadId, required this.objectKey, required this.createdAt, + this.bucketName, + this.awsRegion, }); /// creates new [TransferRecord] object from a [json] map. @@ -40,6 +42,12 @@ class TransferRecord { /// Timestamp of [uploadId] creation. final DateTime createdAt; + /// Amazon S3 bucket name. + final String? bucketName; + + /// AWS region of Amazon S3 bucket. + final String? awsRegion; + /// return json map representation of [TransferRecord] object. Map toJson() => _$TransferRecordToJson(this); diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/transfer_record.g.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/transfer_record.g.dart index 270e70dd8f..d9783e3a7c 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/transfer_record.g.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/transfer/database/transfer_record.g.dart @@ -11,6 +11,8 @@ TransferRecord _$TransferRecordFromJson(Map json) => uploadId: json['uploadId'] as String, objectKey: json['objectKey'] as String, createdAt: DateTime.parse(json['createdAt'] as String), + bucketName: json['bucketName'] as String?, + awsRegion: json['awsRegion'] as String?, ); Map _$TransferRecordToJson(TransferRecord instance) => @@ -18,4 +20,6 @@ Map _$TransferRecordToJson(TransferRecord instance) => 'uploadId': instance.uploadId, 'objectKey': instance.objectKey, 'createdAt': instance.createdAt.toIso8601String(), + 'bucketName': instance.bucketName, + 'awsRegion': instance.awsRegion, }; diff --git a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart index 17015eb5f7..405b5c9bf8 100644 --- a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart +++ b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart @@ -534,6 +534,14 @@ void main() { const StorageUploadDataOptions(), ); registerFallbackValue(const S3DataPayload.empty()); + registerFallbackValue( + const StorageBucket.fromBucketInfo( + BucketInfo( + bucketName: 'bucketName', + region: 'region', + ), + ), + ); }); test('should forward default options to StorageS3Service.uploadData API', @@ -631,6 +639,48 @@ void main() { ); }); + test('should forward bucket to StorageS3Service.uploadData API', + () async { + const testBucket = StorageBucket.fromBucketInfo( + BucketInfo( + bucketName: 'test-bucket', + region: 'test-region', + ), + ); + when( + () => storageS3Service.uploadData( + path: testPath, + dataPayload: any(named: 'dataPayload'), + options: any(named: 'options'), + bucket: testBucket, + ), + ).thenAnswer((_) => testS3UploadTask); + + when(() => testS3UploadTask.result).thenAnswer((_) async => testItem); + uploadDataOperation = storageS3Plugin.uploadData( + data: testData, + path: testPath, + bucket: testBucket, + ); + final capturedBucket = verify( + () => storageS3Service.uploadData( + path: testPath, + dataPayload: any(named: 'dataPayload'), + options: any( + named: 'options', + ), + bucket: captureAny( + named: 'bucket', + ), + ), + ).captured.last; + + expect( + capturedBucket, + testBucket, + ); + }); + test('should forward options.metadata to StorageS3Service.uploadData API', () async { const testMetadata = { diff --git a/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/storage_s3_service_test.dart b/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/storage_s3_service_test.dart index aec65db336..0ca3760d5e 100644 --- a/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/storage_s3_service_test.dart +++ b/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/storage_s3_service_test.dart @@ -4,6 +4,7 @@ import 'dart:async'; import 'package:amplify_core/amplify_core.dart' hide PaginatedResult; +import 'package:amplify_core/src/config/amplify_outputs/storage/bucket_outputs.dart'; import 'package:amplify_core/src/config/amplify_outputs/storage/storage_outputs.dart'; import 'package:amplify_storage_s3_dart/amplify_storage_s3_dart.dart'; import 'package:amplify_storage_s3_dart/src/exception/s3_storage_exception.dart'; @@ -25,9 +26,18 @@ const testPath = StoragePath.fromString('some/path.txt'); void main() { group('StorageS3Service', () { const testBucket = 'bucket1'; + const testBucketName = 'bucket1-name'; const testRegion = 'west-2'; - const storageOutputs = - StorageOutputs(bucketName: testBucket, awsRegion: testRegion); + const testBuckets = BucketOutputs( + name: testBucket, + bucketName: testBucketName, + awsRegion: testRegion, + ); + const storageOutputs = StorageOutputs( + bucketName: testBucket, + awsRegion: testRegion, + buckets: [testBuckets], + ); final pathResolver = TestPathResolver(); late DependencyManager dependencyManager; @@ -35,14 +45,20 @@ void main() { late StorageS3Service storageS3Service; late AWSLogger logger; late AWSSigV4Signer awsSigV4Signer; + late AmplifyUserAgent mockUserAgent; + late AWSHttpClient mockAwsHttpClient; setUp(() { s3Client = MockS3Client(); logger = MockAWSLogger(); awsSigV4Signer = MockAWSSigV4Signer(); + mockUserAgent = MockAmplifyUserAgent(); + mockAwsHttpClient = MockAWSHttpClient(); dependencyManager = DependencyManager() ..addInstance(s3Client) - ..addInstance(awsSigV4Signer); + ..addInstance(awsSigV4Signer) + ..addInstance(mockUserAgent) + ..addInstance(mockAwsHttpClient); storageS3Service = StorageS3Service( storageOutputs: storageOutputs, pathResolver: pathResolver, @@ -69,6 +85,19 @@ void main() { expect(message, contains('Since your bucket name contains dots')); }); + test('creates and caches s3 client info for each storage bucket', () { + final client1 = storageS3Service.getS3ClientInfo( + storageBucket: const StorageBucket.fromBucketInfo( + BucketInfo(bucketName: testBucketName, region: testRegion), + ), + ); + final client2 = storageS3Service.getS3ClientInfo( + storageBucket: StorageBucket.fromOutputs(testBucket), + ); + + expect(client1, client2); + }); + group('list() API', () { late S3ListResult listResult; const testNextContinuationToken = 'get-next-page'; diff --git a/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/task/s3_upload_task_test.dart b/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/task/s3_upload_task_test.dart index 51d373d0c5..7da7824619 100644 --- a/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/task/s3_upload_task_test.dart +++ b/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/task/s3_upload_task_test.dart @@ -28,6 +28,7 @@ void main() { late AWSLogger logger; late transfer.TransferDatabase transferDatabase; const testBucket = 'fake-bucket'; + const testRegion = 'test-region'; const defaultS3ClientConfig = smithy_aws.S3ClientConfig(); final pathResolver = TestPathResolver(); const testUploadDataOptions = StorageUploadDataOptions(); @@ -113,9 +114,10 @@ void main() { final uploadDataTask = S3UploadTask.fromDataPayload( testDataPayload, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: testPath, options: const StorageUploadDataOptions(), logger: logger, @@ -172,9 +174,10 @@ void main() { final uploadDataTask = S3UploadTask.fromDataPayload( testDataPayload, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: testPath, options: testUploadDataOptions, logger: logger, @@ -222,9 +225,10 @@ void main() { final uploadDataTask = S3UploadTask.fromDataPayload( testDataPayloadBytes, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: testPath, options: testUploadDataOptions, logger: logger, @@ -291,9 +295,10 @@ void main() { final uploadDataTask = S3UploadTask.fromDataPayload( testDataPayload, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: testPath, options: testUploadDataOptions, logger: logger, @@ -334,9 +339,10 @@ void main() { final uploadDataTask = S3UploadTask.fromDataPayload( testDataPayload, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: testPath, options: testUploadDataOptions, logger: logger, @@ -369,9 +375,10 @@ void main() { final uploadDataTask = S3UploadTask.fromDataPayload( testDataPayload, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: testPath, options: testUploadDataOptions, logger: logger, @@ -414,9 +421,10 @@ void main() { final uploadDataTask = S3UploadTask.fromDataPayload( testDataPayload, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: testPath, options: testUploadDataOptions, logger: logger, @@ -466,9 +474,10 @@ void main() { final uploadDataTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -526,9 +535,10 @@ void main() { final uploadDataTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -582,9 +592,10 @@ void main() { final uploadDataTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -636,9 +647,10 @@ void main() { final uploadDataTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -774,9 +786,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -956,9 +969,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -1048,9 +1062,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFileWithoutContentType, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -1149,9 +1164,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -1186,9 +1202,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -1218,9 +1235,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testBadFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -1349,9 +1367,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -1393,9 +1412,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: const StorageUploadDataOptions(), logger: logger, @@ -1436,9 +1456,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -1479,9 +1500,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -1560,9 +1582,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -1652,9 +1675,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: const StorageUploadDataOptions(), logger: logger, @@ -1743,9 +1767,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -1813,9 +1838,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -1969,9 +1995,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -2027,9 +2054,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( testLocalFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger, @@ -2091,10 +2119,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( AWSFile.fromPath('fake/file.jpg'), s3Client: s3Client, - defaultS3ClientConfig: - const smithy_aws.S3ClientConfig(usePathStyle: true), + s3ClientConfig: const smithy_aws.S3ClientConfig(usePathStyle: true), pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: const StorageUploadDataOptions( pluginOptions: S3UploadDataPluginOptions( diff --git a/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/transfer/database_html_test.dart b/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/transfer/database_html_test.dart index 73ae16eb75..9a92df808e 100644 --- a/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/transfer/database_html_test.dart +++ b/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/transfer/database_html_test.dart @@ -12,12 +12,16 @@ void main() { group('TransferDatabase for web', () { const testUploadId = 'test-upload-Id'; const testObjectKey = 'test-object-Key'; + const testBucketName = 'test-bucket-name'; + const testAwsRegion = 'test-aws-region'; final testCreatedAt = DateTime(2022, 1, 1); final testTransferRecord = TransferRecord( uploadId: testUploadId, objectKey: testObjectKey, createdAt: testCreatedAt, + bucketName: testBucketName, + awsRegion: testAwsRegion, ); final testTransferRecordJsonString = testTransferRecord.toJsonString(); diff --git a/packages/storage/amplify_storage_s3_dart/test/test_utils/mocks.dart b/packages/storage/amplify_storage_s3_dart/test/test_utils/mocks.dart index 1136704cde..1954fb75fa 100644 --- a/packages/storage/amplify_storage_s3_dart/test/test_utils/mocks.dart +++ b/packages/storage/amplify_storage_s3_dart/test/test_utils/mocks.dart @@ -27,3 +27,7 @@ class MockS3UploadTask extends Mock implements S3UploadTask {} class MockTransferDatabase extends Mock implements TransferDatabase {} class MockSmithyOperation extends Mock implements SmithyOperation {} + +class MockAmplifyUserAgent extends Mock implements AmplifyUserAgent {} + +class MockAWSHttpClient extends Mock implements AWSHttpClient {} From 9d4f747238332b94218b00157c43b14f168dd1af Mon Sep 17 00:00:00 2001 From: Ekjot <43255916+ekjotmultani@users.noreply.github.com> Date: Thu, 24 Oct 2024 09:07:01 -0700 Subject: [PATCH 14/25] feat(storage): multi bucket get properties api (#5577) --- .../category/amplify_storage_category.dart | 1 + .../types/storage/get_properties_options.dart | 9 +- .../integration_test/get_properties_test.dart | 114 ++++++++++++++++++ .../lib/src/amplify_storage_s3_dart_impl.dart | 1 + .../service/storage_s3_service_impl.dart | 4 +- .../test/amplify_storage_s3_dart_test.dart | 3 + 6 files changed, 128 insertions(+), 4 deletions(-) diff --git a/packages/amplify_core/lib/src/category/amplify_storage_category.dart b/packages/amplify_core/lib/src/category/amplify_storage_category.dart index c3f4cfbe2f..84c9603c03 100644 --- a/packages/amplify_core/lib/src/category/amplify_storage_category.dart +++ b/packages/amplify_core/lib/src/category/amplify_storage_category.dart @@ -153,6 +153,7 @@ class StorageCategory extends AmplifyCategory { data: data, onProgress: onProgress, options: options, + bucket: bucket, ), ); } diff --git a/packages/amplify_core/lib/src/types/storage/get_properties_options.dart b/packages/amplify_core/lib/src/types/storage/get_properties_options.dart index bef609f7a7..dbc5b8c112 100644 --- a/packages/amplify_core/lib/src/types/storage/get_properties_options.dart +++ b/packages/amplify_core/lib/src/types/storage/get_properties_options.dart @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import 'package:aws_common/aws_common.dart'; +import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.get_properties_options} /// Configurable options for `Amplify.Storage.getProperties`. @@ -14,13 +14,17 @@ class StorageGetPropertiesOptions /// {@macro amplify_core.storage.get_properties_options} const StorageGetPropertiesOptions({ this.pluginOptions, + this.bucket, }); /// {@macro amplify_core.storage.download_get_properties_plugin_options} final StorageGetPropertiesPluginOptions? pluginOptions; + /// Optionally specify which bucket to retrieve + final StorageBucket? bucket; + @override - List get props => [pluginOptions]; + List get props => [pluginOptions, bucket]; @override String get runtimeTypeName => 'StorageGetPropertiesOptions'; @@ -28,6 +32,7 @@ class StorageGetPropertiesOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), + 'bucket': bucket, }; } diff --git a/packages/storage/amplify_storage_s3/example/integration_test/get_properties_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/get_properties_test.dart index 9540e49650..06116cbb75 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/get_properties_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/get_properties_test.dart @@ -101,5 +101,119 @@ void main() { expect(result.storageItem.size, data.length); }); }); + group('multibucket config', () { + final mainBucket = + StorageBucket.fromOutputs('Storage Integ Test main bucket'); + final secondaryBucket = + StorageBucket.fromOutputs('Storage Integ Test secondary bucket'); + setUpAll(() async { + await configure(amplifyEnvironments['main']!); + addTearDownPath(StoragePath.fromString(path)); + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes(data), + path: StoragePath.fromString(path), + options: const StorageUploadDataOptions(metadata: metadata), + bucket: mainBucket, + ).result; + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes(data), + path: StoragePath.fromString(path), + options: const StorageUploadDataOptions(metadata: metadata), + bucket: secondaryBucket, + ).result; + }); + + testWidgets('String StoragePath', (_) async { + final result = await Amplify.Storage.getProperties( + path: StoragePath.fromString(path), + options: StorageGetPropertiesOptions( + bucket: mainBucket, + ), + ).result; + expect(result.storageItem.path, path); + expect(result.storageItem.metadata, metadata); + expect(result.storageItem.eTag, isNotNull); + expect(result.storageItem.size, data.length); + + final resultSecondaryBucket = await Amplify.Storage.getProperties( + path: StoragePath.fromString(path), + options: StorageGetPropertiesOptions( + bucket: secondaryBucket, + ), + ).result; + expect(resultSecondaryBucket.storageItem.path, path); + expect(resultSecondaryBucket.storageItem.metadata, metadata); + expect(resultSecondaryBucket.storageItem.eTag, isNotNull); + expect(resultSecondaryBucket.storageItem.size, data.length); + }); + + testWidgets('with identity ID', (_) async { + final userIdentityId = await signInNewUser(); + final name = 'get-properties-with-identity-id-${uuid()}'; + final data = 'with identity ID'.codeUnits; + final expectedResolvedPath = 'private/$userIdentityId/$name'; + addTearDownPath(StoragePath.fromString(expectedResolvedPath)); + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes(data), + path: StoragePath.fromString(expectedResolvedPath), + options: const StorageUploadDataOptions(metadata: metadata), + bucket: secondaryBucket, + ).result; + final result = await Amplify.Storage.getProperties( + path: StoragePath.fromIdentityId( + ((identityId) => 'private/$identityId/$name'), + ), + options: StorageGetPropertiesOptions( + bucket: secondaryBucket, + ), + ).result; + expect(result.storageItem.path, expectedResolvedPath); + expect(result.storageItem.metadata, metadata); + expect(result.storageItem.eTag, isNotNull); + expect(result.storageItem.size, data.length); + }); + + testWidgets('not existent path', (_) async { + // we expect StorageNotFoundException here since there is no data uploaded to either bucket on this path + await expectLater( + () => Amplify.Storage.getProperties( + path: const StoragePath.fromString('public/not-existent-path'), + options: StorageGetPropertiesOptions( + bucket: mainBucket, + ), + ).result, + throwsA(isA()), + ); + await expectLater( + () => Amplify.Storage.getProperties( + path: const StoragePath.fromString('public/not-existent-path'), + options: StorageGetPropertiesOptions( + bucket: secondaryBucket, + ), + ).result, + throwsA(isA()), + ); + }); + testWidgets('unauthorized path', (_) async { + await expectLater( + () => Amplify.Storage.getProperties( + path: const StoragePath.fromString('unauthorized/path'), + options: StorageGetPropertiesOptions( + bucket: mainBucket, + ), + ).result, + throwsA(isA()), + ); + await expectLater( + () => Amplify.Storage.getProperties( + path: const StoragePath.fromString('unauthorized/path'), + options: StorageGetPropertiesOptions( + bucket: secondaryBucket, + ), + ).result, + throwsA(isA()), + ); + }); + }); }); } diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index f4ec4035c4..5134b7a704 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -163,6 +163,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface final s3Options = StorageGetPropertiesOptions( pluginOptions: s3PluginOptions, + bucket: options?.bucket, ); return S3GetPropertiesOperation( diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index 86a355eac9..a6feeb334d 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -197,12 +197,12 @@ class StorageS3Service { required StorageGetPropertiesOptions options, }) async { final resolvedPath = await _pathResolver.resolvePath(path: path); - + final s3ClientInfo = getS3ClientInfo(storageBucket: options.bucket); return S3GetPropertiesResult( storageItem: S3Item.fromHeadObjectOutput( await headObject( s3client: _defaultS3Client, - bucket: _storageOutputs.bucketName, + bucket: s3ClientInfo.bucketName, key: resolvedPath, ), path: resolvedPath, diff --git a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart index 405b5c9bf8..3968db24ed 100644 --- a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart +++ b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart @@ -245,6 +245,9 @@ void main() { () async { const testOptions = StorageGetPropertiesOptions( pluginOptions: S3GetPropertiesPluginOptions(), + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), ); when( From 8b7e5b866c081e7d39398f7a454d199fd5479b0b Mon Sep 17 00:00:00 2001 From: Elijah Quartey Date: Mon, 28 Oct 2024 14:04:42 -0500 Subject: [PATCH 15/25] feat(storage): multi bucket remove (#5598) --- .../lib/src/types/storage/remove_options.dart | 9 +- .../example/integration_test/remove_test.dart | 110 ++++++++++++++++++ .../integration_test/utils/object_exists.dart | 7 +- .../integration_test/utils/tear_down.dart | 21 ++++ .../lib/src/amplify_storage_s3_dart_impl.dart | 1 + .../service/storage_s3_service_impl.dart | 7 +- 6 files changed, 148 insertions(+), 7 deletions(-) diff --git a/packages/amplify_core/lib/src/types/storage/remove_options.dart b/packages/amplify_core/lib/src/types/storage/remove_options.dart index f898e77190..95be578c1c 100644 --- a/packages/amplify_core/lib/src/types/storage/remove_options.dart +++ b/packages/amplify_core/lib/src/types/storage/remove_options.dart @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import 'package:aws_common/aws_common.dart'; +import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.remove_options} /// Configurable options for `Amplify.Storage.remove`. @@ -14,13 +14,17 @@ class StorageRemoveOptions /// {@macro amplify_core.storage.remove_options} const StorageRemoveOptions({ this.pluginOptions, + this.bucket, }); /// {@macro amplify_core.storage.remove_plugin_options} final StorageRemovePluginOptions? pluginOptions; + /// Optionally specify which bucket to target + final StorageBucket? bucket; + @override - List get props => [pluginOptions]; + List get props => [pluginOptions, bucket]; @override String get runtimeTypeName => 'StorageRemoveOptions'; @@ -28,6 +32,7 @@ class StorageRemoveOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), + 'bucket': bucket, }; } diff --git a/packages/storage/amplify_storage_s3/example/integration_test/remove_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/remove_test.dart index a46d68e66e..6a8d441be7 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/remove_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/remove_test.dart @@ -64,6 +64,116 @@ void main() { }); }); + group('Multi-bucket', () { + final mainBucket = StorageBucket.fromOutputs( + 'Storage Integ Test main bucket', + ); + final secondaryBucket = StorageBucket.fromOutputs( + 'Storage Integ Test secondary bucket', + ); + final path = 'public/multi-bucket-remove-${uuid()}'; + final storagePath = StoragePath.fromString(path); + setUp(() async { + // upload to main bucket + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes('data'.codeUnits), + path: storagePath, + bucket: mainBucket, + ).result; + }); + + testWidgets('removes from multiple buckets', (_) async { + expect( + await objectExists( + storagePath, + bucket: mainBucket, + ), + true, + ); + + // upload to secondary bucket + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes('data'.codeUnits), + path: storagePath, + bucket: secondaryBucket, + ).result; + + expect( + await objectExists( + storagePath, + bucket: secondaryBucket, + ), + true, + ); + + final mainResult = await Amplify.Storage.remove( + path: storagePath, + options: StorageRemoveOptions(bucket: mainBucket), + ).result; + expect(mainResult.removedItem.path, path); + + // Assert path was only removed from the main bucket + expect( + await objectExists( + storagePath, + bucket: mainBucket, + ), + false, + ); + expect( + await objectExists( + storagePath, + bucket: secondaryBucket, + ), + true, + ); + + final secondaryResult = await Amplify.Storage.remove( + path: storagePath, + options: StorageRemoveOptions(bucket: secondaryBucket), + ).result; + expect(secondaryResult.removedItem.path, path); + expect( + await objectExists( + storagePath, + bucket: secondaryBucket, + ), + false, + ); + }); + + testWidgets('removes when present in bucket', (_) async { + expect( + await objectExists( + storagePath, + bucket: mainBucket, + ), + true, + ); + final mainResult = await Amplify.Storage.remove( + path: storagePath, + options: StorageRemoveOptions(bucket: mainBucket), + ).result; + expect(mainResult.removedItem.path, path); + expect( + await objectExists( + storagePath, + bucket: mainBucket, + ), + false, + ); + + await expectLater( + Amplify.Storage.remove( + path: storagePath, + options: StorageRemoveOptions(bucket: secondaryBucket), + ).result, + completes, + reason: 'non existent path does not throw', + ); + }); + }); + testWidgets('unauthorized path', (_) async { await expectLater( () => Amplify.Storage.remove( diff --git a/packages/storage/amplify_storage_s3/example/integration_test/utils/object_exists.dart b/packages/storage/amplify_storage_s3/example/integration_test/utils/object_exists.dart index c6e533f5ab..aa3e98f97e 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/utils/object_exists.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/utils/object_exists.dart @@ -1,9 +1,12 @@ import 'package:amplify_core/amplify_core.dart'; /// Returns true if an object exists at the given [path]. -Future objectExists(StoragePath path) async { +Future objectExists(StoragePath path, {StorageBucket? bucket}) async { try { - await Amplify.Storage.getProperties(path: path).result; + await Amplify.Storage.getProperties( + path: path, + options: StorageGetPropertiesOptions(bucket: bucket), + ).result; return true; } on StorageNotFoundException { return false; diff --git a/packages/storage/amplify_storage_s3/example/integration_test/utils/tear_down.dart b/packages/storage/amplify_storage_s3/example/integration_test/utils/tear_down.dart index 5c9bd036f2..319cc05ee9 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/utils/tear_down.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/utils/tear_down.dart @@ -36,6 +36,27 @@ void addTearDownPaths(List paths) { ); } +/// Adds a tear down to remove the same object in multiple [buckets]. +void addTearDownMultiBucket(StoragePath path, List buckets) { + addTearDown( + () { + try { + return Future.wait( + buckets.map( + (bucket) => Amplify.Storage.remove( + path: path, + options: StorageRemoveOptions(bucket: bucket), + ).result, + ), + ); + } on Exception catch (e) { + _logger.warn('Failed to remove files after test', e); + rethrow; + } + }, + ); +} + /// Adds a tear down to delete the current user. void addTearDownCurrentUser() { addTearDown(() { diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index 5134b7a704..cfca3bc0e4 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -388,6 +388,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface final s3Options = StorageRemoveOptions( pluginOptions: s3PluginOptions, + bucket: options?.bucket, ); return S3RemoveOperation( diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index a6feeb334d..5848fa3ecd 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -201,7 +201,7 @@ class StorageS3Service { return S3GetPropertiesResult( storageItem: S3Item.fromHeadObjectOutput( await headObject( - s3client: _defaultS3Client, + s3client: s3ClientInfo.client, bucket: s3ClientInfo.bucketName, key: resolvedPath, ), @@ -456,11 +456,12 @@ class StorageS3Service { required StoragePath path, required StorageRemoveOptions options, }) async { + final s3ClientInfo = getS3ClientInfo(storageBucket: options.bucket); final resolvedPath = await _pathResolver.resolvePath(path: path); await _deleteObject( - s3client: _defaultS3Client, - bucket: _storageOutputs.bucketName, + s3client: s3ClientInfo.client, + bucket: s3ClientInfo.bucketName, key: resolvedPath, ); From 670f8eb05527b5e1ede4ebb14dc09bc55926d6eb Mon Sep 17 00:00:00 2001 From: Elijah Quartey Date: Mon, 28 Oct 2024 14:21:30 -0500 Subject: [PATCH 16/25] feat(storage): multi bucket download data (#5599) --- .../types/storage/download_data_options.dart | 9 +++++++-- .../integration_test/download_data_test.dart | 18 ++++++++++++++++++ .../lib/src/amplify_storage_s3_dart_impl.dart | 1 + .../service/storage_s3_service_impl.dart | 7 ++++--- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/packages/amplify_core/lib/src/types/storage/download_data_options.dart b/packages/amplify_core/lib/src/types/storage/download_data_options.dart index 25c59523f1..020856dfe1 100644 --- a/packages/amplify_core/lib/src/types/storage/download_data_options.dart +++ b/packages/amplify_core/lib/src/types/storage/download_data_options.dart @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import 'package:aws_common/aws_common.dart'; +import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.download_data_options} /// Configurable options for `Amplify.Storage.downloadData`. @@ -14,13 +14,17 @@ class StorageDownloadDataOptions /// {@macro amplify_core.storage.download_data_options} const StorageDownloadDataOptions({ this.pluginOptions, + this.bucket, }); /// {@macro amplify_core.storage.download_data_plugin_options} final StorageDownloadDataPluginOptions? pluginOptions; + /// Optionally specify which bucket to target + final StorageBucket? bucket; + @override - List get props => [pluginOptions]; + List get props => [pluginOptions, bucket]; @override String get runtimeTypeName => 'StorageDownloadDataOptions'; @@ -28,6 +32,7 @@ class StorageDownloadDataOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), + 'bucket': bucket, }; } diff --git a/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart index eb028d3919..a88c3631aa 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart @@ -132,6 +132,24 @@ void main() { expect(utf8.decode(downloadResult.bytes), 'data'); expect(downloadResult.downloadedItem.path, publicPath); }); + + testWidgets('multi bucket', (_) async { + final mainBucket = + StorageBucket.fromOutputs('Storage Integ Test main bucket'); + + // TODO(equartey): Add download check for secondary bucket when upload supports multibucket + final downloadResult = await Amplify.Storage.downloadData( + path: StoragePath.fromIdentityId( + (identityId) => 'private/$identityId/$identityName', + ), + options: StorageDownloadDataOptions(bucket: mainBucket), + ).result; + expect(downloadResult.bytes, identityData); + expect( + downloadResult.downloadedItem.path, + 'private/$userIdentityId/$identityName', + ); + }); }); group('download progress', () { diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index cfca3bc0e4..24bc55dcc2 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -217,6 +217,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface final s3Options = StorageDownloadDataOptions( pluginOptions: s3PluginOptions, + bucket: options?.bucket, ); final bytes = BytesBuilder(); diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index 5848fa3ecd..72fe242f80 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -299,10 +299,11 @@ class StorageS3Service { FutureOr Function()? onDone, FutureOr Function()? onError, }) { + final s3ClientInfo = getS3ClientInfo(storageBucket: options.bucket); final downloadDataTask = S3DownloadTask( - s3Client: _defaultS3Client, - defaultS3ClientConfig: _defaultS3ClientConfig, - bucket: _storageOutputs.bucketName, + s3Client: s3ClientInfo.client, + defaultS3ClientConfig: s3ClientInfo.config, + bucket: s3ClientInfo.bucketName, path: path, options: options, pathResolver: _pathResolver, From f9083d4f766e4af3c2b7c541de1fc8dd07c21b1c Mon Sep 17 00:00:00 2001 From: Elijah Quartey Date: Tue, 29 Oct 2024 08:41:01 -0500 Subject: [PATCH 17/25] feat(storage): multi bucket upload file (#5600) --- .../types/storage/upload_file_options.dart | 9 ++- .../integration_test/download_data_test.dart | 33 ++++++-- .../integration_test/upload_file_test.dart | 78 +++++++++++++++++++ .../integration_test/utils/tear_down.dart | 7 +- .../lib/src/amplify_storage_s3_dart_impl.dart | 1 + .../service/storage_s3_service_impl.dart | 7 +- .../test/amplify_storage_s3_dart_test.dart | 16 +++- 7 files changed, 137 insertions(+), 14 deletions(-) diff --git a/packages/amplify_core/lib/src/types/storage/upload_file_options.dart b/packages/amplify_core/lib/src/types/storage/upload_file_options.dart index bad1529468..6b025257fb 100644 --- a/packages/amplify_core/lib/src/types/storage/upload_file_options.dart +++ b/packages/amplify_core/lib/src/types/storage/upload_file_options.dart @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import 'package:aws_common/aws_common.dart'; +import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.upload_file_options} /// Configurable options for `Amplify.Storage.uploadFile`. @@ -15,6 +15,7 @@ class StorageUploadFileOptions const StorageUploadFileOptions({ this.metadata = const {}, this.pluginOptions, + this.bucket, }); /// The metadata attached to the object to be uploaded. @@ -23,8 +24,11 @@ class StorageUploadFileOptions /// {@macro amplify_core.storage.upload_file_plugin_options} final StorageUploadFilePluginOptions? pluginOptions; + /// Optionally specify which bucket to target + final StorageBucket? bucket; + @override - List get props => [metadata, pluginOptions]; + List get props => [metadata, pluginOptions, bucket]; @override String get runtimeTypeName => 'StorageUploadFileOptions'; @@ -33,6 +37,7 @@ class StorageUploadFileOptions Map toJson() => { 'metadata': metadata, 'pluginOptions': pluginOptions?.toJson(), + 'bucket': bucket, }; } diff --git a/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart index a88c3631aa..96dd7c7c36 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart @@ -136,18 +136,39 @@ void main() { testWidgets('multi bucket', (_) async { final mainBucket = StorageBucket.fromOutputs('Storage Integ Test main bucket'); + final secondaryBucket = StorageBucket.fromOutputs( + 'Storage Integ Test secondary bucket', + ); + await Amplify.Storage.uploadData( + path: StoragePath.fromString(publicPath), + data: StorageDataPayload.bytes(bytesData), + bucket: secondaryBucket, + ).result; - // TODO(equartey): Add download check for secondary bucket when upload supports multibucket final downloadResult = await Amplify.Storage.downloadData( - path: StoragePath.fromIdentityId( - (identityId) => 'private/$identityId/$identityName', - ), + path: StoragePath.fromString(publicPath), options: StorageDownloadDataOptions(bucket: mainBucket), ).result; - expect(downloadResult.bytes, identityData); + expect( + downloadResult.bytes, + bytesData, + ); expect( downloadResult.downloadedItem.path, - 'private/$userIdentityId/$identityName', + publicPath, + ); + + final downloadSecondaryResult = await Amplify.Storage.downloadData( + path: StoragePath.fromString(publicPath), + options: StorageDownloadDataOptions(bucket: secondaryBucket), + ).result; + expect( + downloadSecondaryResult.bytes, + bytesData, + ); + expect( + downloadSecondaryResult.downloadedItem.path, + publicPath, ); }); }); diff --git a/packages/storage/amplify_storage_s3/example/integration_test/upload_file_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/upload_file_test.dart index 6e1eb0581e..9113a3c4c3 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/upload_file_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/upload_file_test.dart @@ -11,6 +11,7 @@ import 'package:integration_test/integration_test.dart'; import 'utils/configure.dart'; import 'utils/create_file/create_file.dart'; +import 'utils/object_exists.dart'; import 'utils/sign_in_new_user.dart'; import 'utils/tear_down.dart'; @@ -220,6 +221,83 @@ void main() { }); }); + group('multi-bucket', () { + final mainBucket = + StorageBucket.fromOutputs('Storage Integ Test main bucket'); + final secondaryBucket = StorageBucket.fromOutputs( + 'Storage Integ Test secondary bucket', + ); + + testWidgets('uploads to multiple buckets', (_) async { + final fileId = uuid(); + final path = 'public/multi-bucket-upload-file-$fileId'; + final storagePath = StoragePath.fromString(path); + const content = 'upload file'; + final data = content.codeUnits; + final filePath = await createFile(path: fileId, content: content); + addTearDownMultiBucket( + storagePath, + [mainBucket, secondaryBucket], + ); + // main bucket + final mainResult = await Amplify.Storage.uploadFile( + localFile: AWSFile.fromPath(filePath), + path: storagePath, + options: StorageUploadFileOptions( + pluginOptions: const S3UploadFilePluginOptions( + useAccelerateEndpoint: true, + ), + bucket: mainBucket, + ), + ).result; + expect(mainResult.uploadedItem.path, path); + + final downloadMainResult = await Amplify.Storage.downloadData( + path: storagePath, + options: StorageDownloadDataOptions( + bucket: mainBucket, + ), + ).result; + expect(downloadMainResult.bytes, data); + + // secondary bucket + final secondaryResult = await Amplify.Storage.uploadFile( + localFile: AWSFile.fromPath(filePath), + path: storagePath, + options: StorageUploadFileOptions( + pluginOptions: const S3UploadFilePluginOptions( + useAccelerateEndpoint: true, + ), + bucket: secondaryBucket, + ), + ).result; + expect(secondaryResult.uploadedItem.path, path); + + final downloadSecondaryResult = await Amplify.Storage.downloadData( + path: storagePath, + options: StorageDownloadDataOptions( + bucket: secondaryBucket, + ), + ).result; + expect(downloadSecondaryResult.bytes, data); + + expect( + await objectExists( + storagePath, + bucket: mainBucket, + ), + true, + ); + expect( + await objectExists( + storagePath, + bucket: secondaryBucket, + ), + true, + ); + }); + }); + group('upload progress', () { testWidgets('reports progress', (_) async { final fileId = uuid(); diff --git a/packages/storage/amplify_storage_s3/example/integration_test/utils/tear_down.dart b/packages/storage/amplify_storage_s3/example/integration_test/utils/tear_down.dart index 319cc05ee9..21cddc0675 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/utils/tear_down.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/utils/tear_down.dart @@ -7,11 +7,14 @@ import 'package:flutter_test/flutter_test.dart'; final _logger = AmplifyLogger().createChild('StorageTests'); /// Adds a tear down to remove the object at [path]. -void addTearDownPath(StoragePath path) { +void addTearDownPath(StoragePath path, {StorageBucket? bucket}) { addTearDown( () { try { - return Amplify.Storage.remove(path: path).result; + return Amplify.Storage.remove( + path: path, + options: StorageRemoveOptions(bucket: bucket), + ).result; } on Exception catch (e) { _logger.warn('Failed to remove file after test', e); rethrow; diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index 24bc55dcc2..3b66d88145 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -324,6 +324,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface final s3Options = StorageUploadFileOptions( metadata: options?.metadata ?? const {}, pluginOptions: s3PluginOptions, + bucket: options?.bucket, ); final uploadTask = storageS3Service.uploadFile( diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index 72fe242f80..4ef4417ae8 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -362,6 +362,7 @@ class StorageS3Service { FutureOr Function()? onDone, FutureOr Function()? onError, }) { + final s3ClientInfo = getS3ClientInfo(storageBucket: options.bucket); final s3PluginOptions = options.pluginOptions as S3UploadFilePluginOptions? ?? const S3UploadFilePluginOptions(); @@ -374,9 +375,9 @@ class StorageS3Service { ); final uploadDataTask = S3UploadTask.fromAWSFile( localFile, - s3Client: _defaultS3Client, - s3ClientConfig: _defaultS3ClientConfig, - bucket: _storageOutputs.bucketName, + s3Client: s3ClientInfo.client, + s3ClientConfig: s3ClientInfo.config, + bucket: s3ClientInfo.bucketName, awsRegion: _storageOutputs.awsRegion, path: path, options: uploadDataOptions, diff --git a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart index 3968db24ed..6e2d258b90 100644 --- a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart +++ b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart @@ -415,6 +415,9 @@ void main() { () async { const defaultOptions = StorageDownloadDataOptions( pluginOptions: S3DownloadDataPluginOptions(), + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), ); when( @@ -432,6 +435,7 @@ void main() { downloadDataOperation = storageS3Plugin.downloadData( path: const StoragePath.fromString('public/$testKey'), + options: defaultOptions, ); final capturedOptions = verify( @@ -766,6 +770,9 @@ void main() { () async { const defaultOptions = StorageUploadFileOptions( pluginOptions: S3UploadFilePluginOptions(), + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), ); when( @@ -784,6 +791,7 @@ void main() { uploadFileOperation = storageS3Plugin.uploadFile( path: testPath, localFile: testLocalFile, + options: defaultOptions, ); final capturedParams = verify( @@ -1009,6 +1017,9 @@ void main() { () async { const defaultOptions = StorageRemoveOptions( pluginOptions: S3RemovePluginOptions(), + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), ); when( () => storageS3Service.remove( @@ -1017,7 +1028,10 @@ void main() { ), ).thenAnswer((_) async => testResult); - final removeOperation = storageS3Plugin.remove(path: testPath); + final removeOperation = storageS3Plugin.remove( + path: testPath, + options: defaultOptions, + ); final capturedOptions = verify( () => storageS3Service.remove( From fc1293cdac2d1e303c0ef9dc7a3903822688f32f Mon Sep 17 00:00:00 2001 From: NikaHsn Date: Fri, 1 Nov 2024 10:18:33 -0700 Subject: [PATCH 18/25] chore(storage): add e2e tests for upload data api (#5622) --- .../category/amplify_storage_category.dart | 2 - .../amplify_storage_plugin_interface.dart | 1 - .../types/storage/upload_data_options.dart | 9 +++- .../integration_test/download_data_test.dart | 4 +- .../integration_test/get_properties_test.dart | 18 ++++--- .../example/integration_test/remove_test.dart | 8 ++- .../integration_test/upload_data_test.dart | 53 +++++++++++++++++++ .../lib/src/amplify_storage_s3_dart_impl.dart | 3 +- .../service/storage_s3_service_impl.dart | 3 +- .../test/amplify_storage_s3_dart_test.dart | 48 +++-------------- 10 files changed, 89 insertions(+), 60 deletions(-) diff --git a/packages/amplify_core/lib/src/category/amplify_storage_category.dart b/packages/amplify_core/lib/src/category/amplify_storage_category.dart index 84c9603c03..416446d12e 100644 --- a/packages/amplify_core/lib/src/category/amplify_storage_category.dart +++ b/packages/amplify_core/lib/src/category/amplify_storage_category.dart @@ -144,7 +144,6 @@ class StorageCategory extends AmplifyCategory { required StoragePath path, void Function(StorageTransferProgress)? onProgress, StorageUploadDataOptions? options, - StorageBucket? bucket, }) { return identifyCall( StorageCategoryMethod.uploadData, @@ -153,7 +152,6 @@ class StorageCategory extends AmplifyCategory { data: data, onProgress: onProgress, options: options, - bucket: bucket, ), ); } diff --git a/packages/amplify_core/lib/src/plugin/amplify_storage_plugin_interface.dart b/packages/amplify_core/lib/src/plugin/amplify_storage_plugin_interface.dart index efb891fbef..c4d874d8a0 100644 --- a/packages/amplify_core/lib/src/plugin/amplify_storage_plugin_interface.dart +++ b/packages/amplify_core/lib/src/plugin/amplify_storage_plugin_interface.dart @@ -63,7 +63,6 @@ abstract class StoragePluginInterface extends AmplifyPluginInterface { required StorageDataPayload data, void Function(StorageTransferProgress)? onProgress, StorageUploadDataOptions? options, - StorageBucket? bucket, }) { throw UnimplementedError('uploadData() has not been implemented.'); } diff --git a/packages/amplify_core/lib/src/types/storage/upload_data_options.dart b/packages/amplify_core/lib/src/types/storage/upload_data_options.dart index eb997552de..3b7bd9eb7b 100644 --- a/packages/amplify_core/lib/src/types/storage/upload_data_options.dart +++ b/packages/amplify_core/lib/src/types/storage/upload_data_options.dart @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import 'package:aws_common/aws_common.dart'; +import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.upload_data_options} /// Configurable options for `Amplify.Storage.uploadData`. @@ -15,16 +15,20 @@ class StorageUploadDataOptions const StorageUploadDataOptions({ this.metadata = const {}, this.pluginOptions, + this.bucket, }); /// The metadata attached to the object to be uploaded. final Map metadata; + /// Optionally specify which bucket to target. + final StorageBucket? bucket; + /// {@macro amplify_core.storage.upload_data_plugin_options} final StorageUploadDataPluginOptions? pluginOptions; @override - List get props => [metadata, pluginOptions]; + List get props => [metadata, pluginOptions, bucket]; @override String get runtimeTypeName => 'StorageUploadDataOptions'; @@ -33,6 +37,7 @@ class StorageUploadDataOptions Map toJson() => { 'metadata': metadata, 'pluginOptions': pluginOptions?.toJson(), + 'bucket': bucket, }; } diff --git a/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart index 96dd7c7c36..bf5f7b96f0 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/download_data_test.dart @@ -142,7 +142,9 @@ void main() { await Amplify.Storage.uploadData( path: StoragePath.fromString(publicPath), data: StorageDataPayload.bytes(bytesData), - bucket: secondaryBucket, + options: StorageUploadDataOptions( + bucket: secondaryBucket, + ), ).result; final downloadResult = await Amplify.Storage.downloadData( diff --git a/packages/storage/amplify_storage_s3/example/integration_test/get_properties_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/get_properties_test.dart index 06116cbb75..5bdf3170b0 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/get_properties_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/get_properties_test.dart @@ -112,14 +112,18 @@ void main() { await Amplify.Storage.uploadData( data: StorageDataPayload.bytes(data), path: StoragePath.fromString(path), - options: const StorageUploadDataOptions(metadata: metadata), - bucket: mainBucket, + options: StorageUploadDataOptions( + metadata: metadata, + bucket: mainBucket, + ), ).result; await Amplify.Storage.uploadData( data: StorageDataPayload.bytes(data), path: StoragePath.fromString(path), - options: const StorageUploadDataOptions(metadata: metadata), - bucket: secondaryBucket, + options: StorageUploadDataOptions( + metadata: metadata, + bucket: secondaryBucket, + ), ).result; }); @@ -156,8 +160,10 @@ void main() { await Amplify.Storage.uploadData( data: StorageDataPayload.bytes(data), path: StoragePath.fromString(expectedResolvedPath), - options: const StorageUploadDataOptions(metadata: metadata), - bucket: secondaryBucket, + options: StorageUploadDataOptions( + metadata: metadata, + bucket: secondaryBucket, + ), ).result; final result = await Amplify.Storage.getProperties( path: StoragePath.fromIdentityId( diff --git a/packages/storage/amplify_storage_s3/example/integration_test/remove_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/remove_test.dart index 6a8d441be7..c54822a190 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/remove_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/remove_test.dart @@ -78,7 +78,9 @@ void main() { await Amplify.Storage.uploadData( data: StorageDataPayload.bytes('data'.codeUnits), path: storagePath, - bucket: mainBucket, + options: StorageUploadDataOptions( + bucket: mainBucket, + ), ).result; }); @@ -95,7 +97,9 @@ void main() { await Amplify.Storage.uploadData( data: StorageDataPayload.bytes('data'.codeUnits), path: storagePath, - bucket: secondaryBucket, + options: StorageUploadDataOptions( + bucket: secondaryBucket, + ), ).result; expect( diff --git a/packages/storage/amplify_storage_s3/example/integration_test/upload_data_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/upload_data_test.dart index 89880ae4ef..68fd42aa35 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/upload_data_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/upload_data_test.dart @@ -252,6 +252,59 @@ void main() { }); }); + group('multi-bucket', () { + final mainBucket = + StorageBucket.fromOutputs('Storage Integ Test main bucket'); + final secondaryBucket = StorageBucket.fromOutputs( + 'Storage Integ Test secondary bucket', + ); + + testWidgets('uploads to multiple buckets', (_) async { + final path = 'public/multi-bucket-upload-data-${uuid()}'; + final storagePath = StoragePath.fromString(path); + final data = 'multi bucket upload data byte'.codeUnits; + addTearDownMultiBucket( + storagePath, + [mainBucket, secondaryBucket], + ); + // main bucket + final mainResult = await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes(data), + path: storagePath, + options: StorageUploadDataOptions( + bucket: mainBucket, + ), + ).result; + expect(mainResult.uploadedItem.path, path); + + final downloadMainResult = await Amplify.Storage.downloadData( + path: storagePath, + options: StorageDownloadDataOptions( + bucket: mainBucket, + ), + ).result; + expect(downloadMainResult.bytes, data); + + // secondary bucket + final secondaryResult = await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes(data), + path: storagePath, + options: StorageUploadDataOptions( + bucket: secondaryBucket, + ), + ).result; + expect(secondaryResult.uploadedItem.path, path); + + final downloadSecondaryResult = await Amplify.Storage.downloadData( + path: storagePath, + options: StorageDownloadDataOptions( + bucket: secondaryBucket, + ), + ).result; + expect(downloadSecondaryResult.bytes, data); + }); + }); + group('upload progress', () { testWidgets('reports progress for byte data', (_) async { final path = 'public/upload-data-progress-bytes-${uuid()}'; diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index 3b66d88145..5512a934e2 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -276,7 +276,6 @@ class AmplifyStorageS3Dart extends StoragePluginInterface required StoragePath path, void Function(S3TransferProgress)? onProgress, StorageUploadDataOptions? options, - StorageBucket? bucket, }) { final s3PluginOptions = reifyPluginOptions( pluginOptions: options?.pluginOptions, @@ -285,6 +284,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface final s3Options = StorageUploadDataOptions( metadata: options?.metadata ?? const {}, + bucket: options?.bucket, pluginOptions: s3PluginOptions, ); @@ -293,7 +293,6 @@ class AmplifyStorageS3Dart extends StoragePluginInterface dataPayload: data, options: s3Options, onProgress: onProgress, - bucket: bucket, ); return S3UploadDataOperation( diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index 4ef4417ae8..9140551535 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -329,9 +329,8 @@ class StorageS3Service { void Function(S3TransferProgress)? onProgress, FutureOr Function()? onDone, FutureOr Function()? onError, - StorageBucket? bucket, }) { - final s3ClientInfo = getS3ClientInfo(storageBucket: bucket); + final s3ClientInfo = getS3ClientInfo(storageBucket: options.bucket); final uploadDataTask = S3UploadTask.fromDataPayload( dataPayload, s3Client: s3ClientInfo.client, diff --git a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart index 6e2d258b90..5d01e5736c 100644 --- a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart +++ b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart @@ -610,6 +610,12 @@ void main() { test('should forward options to StorageS3Service.uploadData API', () async { const testOptions = StorageUploadDataOptions( + bucket: StorageBucket.fromBucketInfo( + BucketInfo( + bucketName: 'test-bucket', + region: 'test-region', + ), + ), pluginOptions: S3UploadDataPluginOptions( getProperties: true, useAccelerateEndpoint: true, @@ -646,48 +652,6 @@ void main() { ); }); - test('should forward bucket to StorageS3Service.uploadData API', - () async { - const testBucket = StorageBucket.fromBucketInfo( - BucketInfo( - bucketName: 'test-bucket', - region: 'test-region', - ), - ); - when( - () => storageS3Service.uploadData( - path: testPath, - dataPayload: any(named: 'dataPayload'), - options: any(named: 'options'), - bucket: testBucket, - ), - ).thenAnswer((_) => testS3UploadTask); - - when(() => testS3UploadTask.result).thenAnswer((_) async => testItem); - uploadDataOperation = storageS3Plugin.uploadData( - data: testData, - path: testPath, - bucket: testBucket, - ); - final capturedBucket = verify( - () => storageS3Service.uploadData( - path: testPath, - dataPayload: any(named: 'dataPayload'), - options: any( - named: 'options', - ), - bucket: captureAny( - named: 'bucket', - ), - ), - ).captured.last; - - expect( - capturedBucket, - testBucket, - ); - }); - test('should forward options.metadata to StorageS3Service.uploadData API', () async { const testMetadata = { From 59ebae65e3cfb4de36f303c16a3a324878910dd0 Mon Sep 17 00:00:00 2001 From: Ekjot <43255916+ekjotmultani@users.noreply.github.com> Date: Mon, 4 Nov 2024 15:08:41 -0800 Subject: [PATCH 19/25] feat(storage): multi bucket list api (#5576) * added bucket option to list api options and added tests * updated tests to account for changes to uploadData api --------- Co-authored-by: ekjotmultani Co-authored-by: NikaHsn Co-authored-by: Tyler-Larkin Co-authored-by: Elijah Quartey --- .../lib/src/types/storage/list_options.dart | 9 +- .../example/integration_test/list_test.dart | 111 ++++++++++++++++-- .../lib/src/amplify_storage_s3_dart_impl.dart | 1 + .../service/storage_s3_service_impl.dart | 9 +- .../test/amplify_storage_s3_dart_test.dart | 3 + 5 files changed, 118 insertions(+), 15 deletions(-) diff --git a/packages/amplify_core/lib/src/types/storage/list_options.dart b/packages/amplify_core/lib/src/types/storage/list_options.dart index c046f90d55..f0a1d80962 100644 --- a/packages/amplify_core/lib/src/types/storage/list_options.dart +++ b/packages/amplify_core/lib/src/types/storage/list_options.dart @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import 'package:aws_common/aws_common.dart'; +import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.list_options} /// Configurable options for `Amplify.Storage.list`. @@ -15,6 +15,7 @@ class StorageListOptions const StorageListOptions({ this.pageSize = 1000, this.nextToken, + this.bucket, this.pluginOptions, }); @@ -27,8 +28,11 @@ class StorageListOptions /// {@macro amplify_core.storage.list_plugin_options} final StorageListPluginOptions? pluginOptions; + /// Optionally specify which bucket to retrieve + final StorageBucket? bucket; + @override - List get props => [pageSize, nextToken, pluginOptions]; + List get props => [pageSize, nextToken, pluginOptions, bucket]; @override String get runtimeTypeName => 'StorageListOptions'; @@ -37,6 +41,7 @@ class StorageListOptions Map toJson() => { 'pageSize': pageSize, 'nextToken': nextToken, + 'bucket': bucket, 'pluginOptions': pluginOptions?.toJson(), }; } diff --git a/packages/storage/amplify_storage_s3/example/integration_test/list_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/list_test.dart index 22e2a2766b..15c3737225 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/list_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/list_test.dart @@ -20,18 +20,41 @@ void main() { '$uniquePrefix/file2.txt', '$uniquePrefix/subdir/file3.txt', '$uniquePrefix/subdir2#file4.txt', + '$uniquePrefix/file5.txt', + '$uniquePrefix/file6.txt', + '$uniquePrefix/subdir3/file7.txt', + '$uniquePrefix/subdir4#file8.txt', ]; group('standard config', () { + final mainBucket = + StorageBucket.fromOutputs('Storage Integ Test main bucket'); + final secondaryBucket = StorageBucket.fromOutputs( + 'Storage Integ Test secondary bucket', + ); setUpAll(() async { await configure(amplifyEnvironments['main']!); - - for (final path in uploadedPaths) { + for (var pathIndex = 0; + pathIndex < uploadedPaths.length ~/ 2; + pathIndex++) { await Amplify.Storage.uploadData( - path: StoragePath.fromString(path), + path: StoragePath.fromString(uploadedPaths[pathIndex]), data: StorageDataPayload.bytes('test content'.codeUnits), + options: StorageUploadDataOptions( + bucket: mainBucket, + ), + ).result; + } + for (var pathIndex = uploadedPaths.length ~/ 2; + pathIndex < uploadedPaths.length; + pathIndex++) { + await Amplify.Storage.uploadData( + path: StoragePath.fromString(uploadedPaths[pathIndex]), + data: StorageDataPayload.bytes('test content'.codeUnits), + options: StorageUploadDataOptions( + bucket: secondaryBucket, + ), ).result; } - for (final path in uploadedPaths) { addTearDownPath(StoragePath.fromString(path)); } @@ -39,13 +62,31 @@ void main() { group('list() without options', () { testWidgets('should list all files with unique prefix', (_) async { - final listResult = await Amplify.Storage.list( + // this will use the main bucket by default when no optional bucket is specified + final listResultMainBucket = await Amplify.Storage.list( path: StoragePath.fromString(uniquePrefix), ).result; - - for (final uploadedPath in uploadedPaths) { + final listResultSecondaryBucket = await Amplify.Storage.list( + path: StoragePath.fromString(uniquePrefix), + options: StorageListOptions( + bucket: secondaryBucket, + ), + ).result; + for (var pathIndex = 0; + pathIndex < uploadedPaths.length ~/ 2; + pathIndex++) { + expect( + listResultMainBucket.items + .any((item) => item.path == uploadedPaths[pathIndex]), + isTrue, + ); + } + for (var pathIndex = uploadedPaths.length ~/ 2; + pathIndex < uploadedPaths.length; + pathIndex++) { expect( - listResult.items.any((item) => item.path == uploadedPath), + listResultSecondaryBucket.items + .any((item) => item.path == uploadedPaths[pathIndex]), isTrue, ); } @@ -101,6 +142,17 @@ void main() { ), ).result as S3ListResult; + final listResultSecondaryBucket = await Amplify.Storage.list( + path: StoragePath.fromString('$uniquePrefix/'), + options: StorageListOptions( + pluginOptions: const S3ListPluginOptions( + excludeSubPaths: true, + delimiter: '#', + ), + bucket: secondaryBucket, + ), + ).result as S3ListResult; + expect(listResult.items.length, 3); expect(listResult.items.first.path, contains('file1.txt')); @@ -110,6 +162,19 @@ void main() { '$uniquePrefix/subdir2#', ); expect(listResult.metadata.delimiter, '#'); + + expect(listResultSecondaryBucket.items.length, 3); + expect( + listResultSecondaryBucket.items.first.path, + contains('file5.txt'), + ); + + expect(listResultSecondaryBucket.metadata.subPaths.length, 1); + expect( + listResultSecondaryBucket.metadata.subPaths.first, + '$uniquePrefix/subdir4#', + ); + expect(listResultSecondaryBucket.metadata.delimiter, '#'); }); }); @@ -123,6 +188,20 @@ void main() { expect(listResult.items.length, 2); expect(listResult.items.first.path, contains('file1.txt')); + + final listResultSecondaryBucket = await Amplify.Storage.list( + path: StoragePath.fromString(uniquePrefix), + options: StorageListOptions( + pageSize: 2, + bucket: secondaryBucket, + ), + ).result; + + expect(listResultSecondaryBucket.items.length, 2); + expect( + listResultSecondaryBucket.items.first.path, + contains('file5.txt'), + ); }); testWidgets('should list files with pagination', (_) async { @@ -157,8 +236,22 @@ void main() { ), ).result; - expect(listResult.items.length, uploadedPaths.length); + expect(listResult.items.length, uploadedPaths.length ~/ 2); expect(listResult.nextToken, isNull); + + final listResultSecondaryBucket = await Amplify.Storage.list( + path: StoragePath.fromString(uniquePrefix), + options: StorageListOptions( + pluginOptions: const S3ListPluginOptions.listAll(), + bucket: secondaryBucket, + ), + ).result; + + expect( + listResultSecondaryBucket.items.length, + uploadedPaths.length ~/ 2, + ); + expect(listResultSecondaryBucket.nextToken, isNull); }); }); }); diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index 5512a934e2..41cbcc5df8 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -136,6 +136,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface final s3Options = StorageListOptions( pluginOptions: s3PluginOptions, nextToken: options?.nextToken, + bucket: options?.bucket, pageSize: options?.pageSize ?? 1000, ); diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index 9140551535..783eed55c2 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -125,11 +125,12 @@ class StorageS3Service { const S3ListPluginOptions(); final resolvedPath = await _pathResolver.resolvePath(path: path); + final s3ClientInfo = getS3ClientInfo(storageBucket: options.bucket); if (!s3PluginOptions.listAll) { final request = s3.ListObjectsV2Request.build((builder) { builder - ..bucket = _storageOutputs.bucketName + ..bucket = s3ClientInfo.bucketName ..prefix = resolvedPath ..maxKeys = options.pageSize ..continuationToken = options.nextToken @@ -140,7 +141,7 @@ class StorageS3Service { try { return S3ListResult.fromPaginatedResult( - await _defaultS3Client.listObjectsV2(request).result, + await s3ClientInfo.client.listObjectsV2(request).result, ); } on smithy.UnknownSmithyHttpException catch (error) { // S3Client.headObject may return 403 error @@ -156,14 +157,14 @@ class StorageS3Service { try { final request = s3.ListObjectsV2Request.build((builder) { builder - ..bucket = _storageOutputs.bucketName + ..bucket = s3ClientInfo.bucketName ..prefix = resolvedPath ..delimiter = s3PluginOptions.excludeSubPaths ? s3PluginOptions.delimiter : null; }); - listResult = await _defaultS3Client.listObjectsV2(request).result; + listResult = await s3ClientInfo.client.listObjectsV2(request).result; recursiveResult = S3ListResult.fromPaginatedResult( listResult, ); diff --git a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart index 5d01e5736c..239b97187b 100644 --- a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart +++ b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart @@ -140,6 +140,9 @@ void main() { const testOptions = StorageListOptions( pluginOptions: S3ListPluginOptions(excludeSubPaths: true), nextToken: 'next-token-123', + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), pageSize: 2, ); From ef95d1f7afa6220bfc39df1b0d198ce5f2f1825f Mon Sep 17 00:00:00 2001 From: NikaHsn Date: Wed, 6 Nov 2024 11:14:11 -0800 Subject: [PATCH 20/25] feat(storage): multi bucket remove many api (#5633) --- .../types/storage/remove_many_options.dart | 12 +- .../integration_test/remove_many_test.dart | 121 ++++++++++++++++++ .../lib/src/amplify_storage_s3_dart_impl.dart | 1 + .../service/storage_s3_service_impl.dart | 6 +- .../test/amplify_storage_s3_dart_test.dart | 3 + 5 files changed, 139 insertions(+), 4 deletions(-) diff --git a/packages/amplify_core/lib/src/types/storage/remove_many_options.dart b/packages/amplify_core/lib/src/types/storage/remove_many_options.dart index d9e32131fe..05db2da027 100644 --- a/packages/amplify_core/lib/src/types/storage/remove_many_options.dart +++ b/packages/amplify_core/lib/src/types/storage/remove_many_options.dart @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import 'package:aws_common/aws_common.dart'; +import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.remove_many_options} /// Configurable options for `Amplify.Storage.removeMany`. @@ -14,13 +14,20 @@ class StorageRemoveManyOptions /// {@macro amplify_core.storage.remove_many_options} const StorageRemoveManyOptions({ this.pluginOptions, + this.bucket, }); /// {@macro amplify_core.storage.remove_many_plugin_options} final StorageRemoveManyPluginOptions? pluginOptions; + /// Optionally specify which bucket to target + final StorageBucket? bucket; + @override - List get props => [pluginOptions]; + List get props => [ + pluginOptions, + bucket, + ]; @override String get runtimeTypeName => 'StorageRemoveManyOptions'; @@ -28,6 +35,7 @@ class StorageRemoveManyOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), + 'bucket': bucket, }; } diff --git a/packages/storage/amplify_storage_s3/example/integration_test/remove_many_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/remove_many_test.dart index 87c5ff2e3b..435ed1d986 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/remove_many_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/remove_many_test.dart @@ -90,6 +90,127 @@ void main() { }); }); + group('Multi-bucket', () { + final mainBucket = StorageBucket.fromOutputs( + 'Storage Integ Test main bucket', + ); + final secondaryBucket = StorageBucket.fromOutputs( + 'Storage Integ Test secondary bucket', + ); + final path1 = 'public/multi-bucket-remove-many-${uuid()}'; + final path2 = 'public/multi-bucket-remove-many-${uuid()}'; + final storagePath1 = StoragePath.fromString(path1); + final storagePath2 = StoragePath.fromString(path2); + setUp(() async { + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes('data'.codeUnits), + path: storagePath1, + options: StorageUploadDataOptions( + bucket: mainBucket, + ), + ).result; + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes('data'.codeUnits), + path: storagePath2, + options: StorageUploadDataOptions( + bucket: mainBucket, + ), + ).result; + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes('data'.codeUnits), + path: storagePath1, + options: StorageUploadDataOptions( + bucket: secondaryBucket, + ), + ).result; + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes('data'.codeUnits), + path: storagePath2, + options: StorageUploadDataOptions( + bucket: secondaryBucket, + ), + ).result; + }); + + testWidgets('removes objects from main bucket', (_) async { + expect( + await objectExists( + storagePath1, + bucket: mainBucket, + ), + true, + ); + expect( + await objectExists( + storagePath2, + bucket: mainBucket, + ), + true, + ); + final result = await Amplify.Storage.removeMany( + paths: [storagePath1, storagePath2], + options: StorageRemoveManyOptions( + bucket: mainBucket, + ), + ).result; + expect( + await objectExists( + storagePath1, + bucket: mainBucket, + ), + false, + ); + expect( + await objectExists( + storagePath2, + bucket: mainBucket, + ), + false, + ); + final removedPaths = result.removedItems.map((i) => i.path).toList(); + expect(removedPaths, unorderedEquals([path1, path2])); + }); + + testWidgets('removes objects from secondary bucket', (_) async { + expect( + await objectExists( + storagePath1, + bucket: secondaryBucket, + ), + true, + ); + expect( + await objectExists( + storagePath2, + bucket: secondaryBucket, + ), + true, + ); + final result = await Amplify.Storage.removeMany( + paths: [storagePath1, storagePath2], + options: StorageRemoveManyOptions( + bucket: secondaryBucket, + ), + ).result; + expect( + await objectExists( + storagePath1, + bucket: secondaryBucket, + ), + false, + ); + expect( + await objectExists( + storagePath2, + bucket: secondaryBucket, + ), + false, + ); + final removedPaths = result.removedItems.map((i) => i.path).toList(); + expect(removedPaths, unorderedEquals([path1, path2])); + }); + }); + testWidgets('unauthorized path', (_) async { final result = await Amplify.Storage.removeMany( paths: [const StoragePath.fromString('unauthorized/path')], diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index 41cbcc5df8..346b08a2b9 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -417,6 +417,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface final s3Options = StorageRemoveManyOptions( pluginOptions: s3PluginOptions, + bucket: options?.bucket, ); return S3RemoveManyOperation( diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index 783eed55c2..4cadf2d644 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -491,6 +491,8 @@ class StorageS3Service { final objectIdentifiersToRemove = resolvedPaths.map((path) => s3.ObjectIdentifier(key: path)).toList(); + final s3ClientInfo = getS3ClientInfo(storageBucket: options.bucket); + final removedItems = []; final removedErrors = []; @@ -503,7 +505,7 @@ class StorageS3Service { ); final request = s3.DeleteObjectsRequest.build((builder) { builder - ..bucket = _storageOutputs.bucketName + ..bucket = s3ClientInfo.bucketName // force to use sha256 instead of md5 ..checksumAlgorithm = s3.ChecksumAlgorithm.sha256 ..delete = s3.Delete.build((builder) { @@ -511,7 +513,7 @@ class StorageS3Service { }).toBuilder(); }); try { - final output = await _defaultS3Client.deleteObjects(request).result; + final output = await s3ClientInfo.client.deleteObjects(request).result; removedItems.addAll( output.deleted?.toList().map( (removedObject) => S3Item.fromS3Object( diff --git a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart index 239b97187b..f144a068c1 100644 --- a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart +++ b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart @@ -1118,6 +1118,9 @@ void main() { () async { const testOptions = StorageRemoveManyOptions( pluginOptions: S3RemoveManyPluginOptions(), + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), ); when( From fcc011d4bf51be3b4a9cf206b90c6312ce605284 Mon Sep 17 00:00:00 2001 From: Elijah Quartey Date: Mon, 11 Nov 2024 13:34:19 -0600 Subject: [PATCH 21/25] feat(storage): multi bucket get url (#5659) --- .../src/types/storage/get_url_options.dart | 12 +++- .../integration_test/get_url_test.dart | 56 +++++++++++++++++++ .../lib/src/amplify_storage_s3_dart_impl.dart | 1 + .../service/storage_s3_service_impl.dart | 17 +++--- .../test/amplify_storage_s3_dart_test.dart | 23 ++++---- 5 files changed, 88 insertions(+), 21 deletions(-) diff --git a/packages/amplify_core/lib/src/types/storage/get_url_options.dart b/packages/amplify_core/lib/src/types/storage/get_url_options.dart index 3f4078839e..62954e8281 100644 --- a/packages/amplify_core/lib/src/types/storage/get_url_options.dart +++ b/packages/amplify_core/lib/src/types/storage/get_url_options.dart @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import 'package:aws_common/aws_common.dart'; +import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.get_url_options} /// Configurable options for `Amplify.Storage.getUrl`. @@ -14,13 +14,20 @@ class StorageGetUrlOptions /// {@macro amplify_core.storage.get_url_options} const StorageGetUrlOptions({ this.pluginOptions, + this.bucket, }); /// {@macro amplify_core.storage.get_url_plugin_options} final StorageGetUrlPluginOptions? pluginOptions; + /// Optionally specify which bucket to target + final StorageBucket? bucket; + @override - List get props => [pluginOptions]; + List get props => [ + pluginOptions, + bucket, + ]; @override String get runtimeTypeName => 'StorageGetUrlOptions'; @@ -28,6 +35,7 @@ class StorageGetUrlOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), + 'bucket': bucket, }; } diff --git a/packages/storage/amplify_storage_s3/example/integration_test/get_url_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/get_url_test.dart index 2c26f5b610..4d49674900 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/get_url_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/get_url_test.dart @@ -172,6 +172,62 @@ void main() { expect(actualData, data); }); }); + + group('multi bucket', () { + final mainBucket = StorageBucket.fromOutputs( + 'Storage Integ Test main bucket', + ); + final secondaryBucket = StorageBucket.fromOutputs( + 'Storage Integ Test secondary bucket', + ); + final pathMain = 'public/multi-bucket-get-url-${uuid()}'; + final pathSecondary = 'public/multi-bucket-get-url-${uuid()}'; + final storagePathMain = StoragePath.fromString(pathMain); + final storagePathSecondary = StoragePath.fromString(pathSecondary); + + setUp(() async { + addTearDownPath(storagePathMain); + addTearDownPath(storagePathSecondary); + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes(data), + path: storagePathMain, + options: StorageUploadDataOptions( + bucket: mainBucket, + ), + ).result; + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes(data), + path: storagePathSecondary, + options: StorageUploadDataOptions( + bucket: secondaryBucket, + ), + ).result; + }); + + testWidgets('can get url from main bucket', (_) async { + final result = await Amplify.Storage.getUrl( + path: storagePathMain, + options: StorageGetUrlOptions( + bucket: mainBucket, + ), + ).result; + expect(result.url.path, '/$pathMain'); + final actualData = await readData(result.url); + expect(actualData, data); + }); + + testWidgets('can get url from secondary bucket', (_) async { + final result = await Amplify.Storage.getUrl( + path: storagePathSecondary, + options: StorageGetUrlOptions( + bucket: secondaryBucket, + ), + ).result; + expect(result.url.path, '/$pathSecondary'); + final actualData = await readData(result.url); + expect(actualData, data); + }); + }); }); group('config with dots in name', () { diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index 346b08a2b9..3abc9c7490 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -191,6 +191,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface final s3Options = StorageGetUrlOptions( pluginOptions: s3PluginOptions, + bucket: options?.bucket, ); return S3GetUrlOperation( diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index 4cadf2d644..05b71ea05f 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -222,9 +222,10 @@ class StorageS3Service { }) async { final s3PluginOptions = options.pluginOptions as S3GetUrlPluginOptions? ?? const S3GetUrlPluginOptions(); + final s3ClientInfo = getS3ClientInfo(storageBucket: options.bucket); if (s3PluginOptions.useAccelerateEndpoint && - _defaultS3ClientConfig.usePathStyle) { + s3ClientInfo.config.usePathStyle) { throw s3_exception.accelerateEndpointUnusable; } @@ -234,20 +235,20 @@ class StorageS3Service { // the `getProperties` API (i.e. HeadObject) await getProperties( path: path, - options: const StorageGetPropertiesOptions(), + options: StorageGetPropertiesOptions(bucket: options.bucket), ); } var resolvedPath = await _pathResolver.resolvePath(path: path); var host = - '${_storageOutputs.bucketName}.${_getS3EndpointHost(region: _storageOutputs.awsRegion)}'; - if (_defaultS3ClientConfig.usePathStyle) { - host = host.replaceFirst('${_storageOutputs.bucketName}.', ''); - resolvedPath = '${_storageOutputs.bucketName}/$resolvedPath'; + '${s3ClientInfo.bucketName}.${_getS3EndpointHost(region: s3ClientInfo.awsRegion)}'; + if (s3ClientInfo.config.usePathStyle) { + host = host.replaceFirst('${s3ClientInfo.bucketName}.', ''); + resolvedPath = '${s3ClientInfo.bucketName}/$resolvedPath'; } else if (s3PluginOptions.useAccelerateEndpoint) { // https: //docs.aws.amazon.com/AmazonS3/latest/userguide/transfer-acceleration-getting-started.html host = host - .replaceFirst(RegExp('${_storageOutputs.awsRegion}\\.'), '') + .replaceFirst(RegExp('${s3ClientInfo.awsRegion}\\.'), '') .replaceFirst(RegExp(r'\.s3\.'), '.s3-accelerate.'); } @@ -263,7 +264,7 @@ class StorageS3Service { credentialsProvider: _credentialsProvider, ); final signerScope = sigv4.AWSCredentialScope( - region: _storageOutputs.awsRegion, + region: s3ClientInfo.awsRegion, service: AWSService.s3, ); diff --git a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart index f144a068c1..db95a371d2 100644 --- a/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart +++ b/packages/storage/amplify_storage_s3_dart/test/amplify_storage_s3_dart_test.dart @@ -348,6 +348,9 @@ void main() { expiresIn: Duration(minutes: 10), useAccelerateEndpoint: true, ), + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), ); when( @@ -418,9 +421,6 @@ void main() { () async { const defaultOptions = StorageDownloadDataOptions( pluginOptions: S3DownloadDataPluginOptions(), - bucket: StorageBucket.fromBucketInfo( - BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), - ), ); when( @@ -438,7 +438,6 @@ void main() { downloadDataOperation = storageS3Plugin.downloadData( path: const StoragePath.fromString('public/$testKey'), - options: defaultOptions, ); final capturedOptions = verify( @@ -471,6 +470,9 @@ void main() { useAccelerateEndpoint: true, getProperties: true, ), + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), ); when( @@ -737,9 +739,6 @@ void main() { () async { const defaultOptions = StorageUploadFileOptions( pluginOptions: S3UploadFilePluginOptions(), - bucket: StorageBucket.fromBucketInfo( - BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), - ), ); when( @@ -758,7 +757,6 @@ void main() { uploadFileOperation = storageS3Plugin.uploadFile( path: testPath, localFile: testLocalFile, - options: defaultOptions, ); final capturedParams = verify( @@ -798,6 +796,9 @@ void main() { getProperties: true, useAccelerateEndpoint: true, ), + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), ); when( @@ -984,9 +985,6 @@ void main() { () async { const defaultOptions = StorageRemoveOptions( pluginOptions: S3RemovePluginOptions(), - bucket: StorageBucket.fromBucketInfo( - BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), - ), ); when( () => storageS3Service.remove( @@ -1024,6 +1022,9 @@ void main() { test('should forward options to StorageS3Service.remove() API', () async { const testOptions = StorageRemoveOptions( pluginOptions: S3RemovePluginOptions(), + bucket: StorageBucket.fromBucketInfo( + BucketInfo(bucketName: 'unit-test-bucket', region: 'us-east-2'), + ), ); when( From a487307eafed10fd41452e7670c6d7b0715e258e Mon Sep 17 00:00:00 2001 From: Ekjot <43255916+ekjotmultani@users.noreply.github.com> Date: Tue, 12 Nov 2024 14:09:22 -0800 Subject: [PATCH 22/25] feat(storage): multi bucket download file api (#5656) * Updated dowload file options to accept bucket parameter added integration tests * updated todo message * updates variable names and tests * Update packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart --------- Co-authored-by: NikaHsn --- .../types/storage/download_file_options.dart | 9 ++- .../integration_test/download_file_test.dart | 61 ++++++++++++++++++- .../lib/src/amplify_storage_s3_dart_impl.dart | 1 + .../download_file/download_file_html.dart | 3 +- .../download_file/download_file_io.dart | 1 + 5 files changed, 71 insertions(+), 4 deletions(-) diff --git a/packages/amplify_core/lib/src/types/storage/download_file_options.dart b/packages/amplify_core/lib/src/types/storage/download_file_options.dart index 8681667131..23d745f5e6 100644 --- a/packages/amplify_core/lib/src/types/storage/download_file_options.dart +++ b/packages/amplify_core/lib/src/types/storage/download_file_options.dart @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import 'package:aws_common/aws_common.dart'; +import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.download_file_options} /// Configurable options for `Amplify.Storage.downloadFile`. @@ -14,13 +14,17 @@ class StorageDownloadFileOptions /// {@macro amplify_core.storage.download_file_options} const StorageDownloadFileOptions({ this.pluginOptions, + this.bucket, }); /// {@macro amplify_core.storage.download_file_plugin_options} final StorageDownloadFilePluginOptions? pluginOptions; + /// Optionally specify which bucket to target + final StorageBucket? bucket; + @override - List get props => [pluginOptions]; + List get props => [pluginOptions, bucket]; @override String get runtimeTypeName => 'StorageDownloadFileOptions'; @@ -28,6 +32,7 @@ class StorageDownloadFileOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), + 'bucket': bucket, }; } diff --git a/packages/storage/amplify_storage_s3/example/integration_test/download_file_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/download_file_test.dart index fb3dd7e554..4ed64792e5 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/download_file_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/download_file_test.dart @@ -28,7 +28,8 @@ void main() { final name = 'download-file-with-identity-id-${uuid()}'; final metadataFilePath = 'public/download-file-get-properties-${uuid()}'; final metadata = {'description': 'foo'}; - + final secondaryBucket = + StorageBucket.fromOutputs('Storage Integ Test secondary bucket'); setUpAll(() async { directory = kIsWeb ? '/' : (await getTemporaryDirectory()).path; }); @@ -51,6 +52,22 @@ void main() { ), ).result; + // secondary bucket uploads + + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes(data), + path: StoragePath.fromString(publicPath), + options: StorageUploadDataOptions(bucket: secondaryBucket), + ).result; + + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes(data), + path: StoragePath.fromIdentityId( + (identityId) => 'private/$identityId/$name', + ), + options: StorageUploadDataOptions(bucket: secondaryBucket), + ).result; + await Amplify.Storage.uploadData( data: StorageDataPayload.bytes(data), path: StoragePath.fromString(metadataFilePath), @@ -68,6 +85,48 @@ void main() { ); }); + group('multibucket', () { + testWidgets('to file', (_) async { + final downloadFilePath = '$directory/downloaded-file.txt'; + + final result = await Amplify.Storage.downloadFile( + path: StoragePath.fromString(publicPath), + localFile: AWSFile.fromPath(downloadFilePath), + options: StorageDownloadFileOptions( + bucket: secondaryBucket, + ), + ).result; + + // Web browsers do not grant access to read arbitrary files + if (!kIsWeb) { + final downloadedFile = await readFile(path: downloadFilePath); + expect(downloadedFile, data); + } + + expect(result.localFile.path, downloadFilePath); + expect(result.downloadedItem.path, publicPath); + }); + testWidgets('from identity ID', (_) async { + final downloadFilePath = '$directory/downloaded-file.txt'; + final result = await Amplify.Storage.downloadFile( + path: StoragePath.fromIdentityId( + (identityId) => 'private/$identityId/$name', + ), + localFile: AWSFile.fromPath(downloadFilePath), + options: StorageDownloadFileOptions( + bucket: secondaryBucket, + ), + ).result; + + if (!kIsWeb) { + final downloadedFile = await readFile(path: downloadFilePath); + expect(downloadedFile, data); + } + expect(result.localFile.path, downloadFilePath); + expect(result.downloadedItem.path, identityPath); + }); + }); + group('for file type', () { testWidgets('to file', (_) async { final downloadFilePath = '$directory/downloaded-file.txt'; diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index 3abc9c7490..f754941b43 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -260,6 +260,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface ); options = StorageDownloadFileOptions( pluginOptions: s3PluginOptions, + bucket: options?.bucket, ); return download_file_impl.downloadFile( path: path, diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/platform_impl/download_file/download_file_html.dart b/packages/storage/amplify_storage_s3_dart/lib/src/platform_impl/download_file/download_file_html.dart index 32f79a9d38..a77ee6f8c5 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/platform_impl/download_file/download_file_html.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/platform_impl/download_file/download_file_html.dart @@ -58,7 +58,7 @@ Future _downloadFromUrl({ // operation. final downloadedItem = (await storageS3Service.getProperties( path: path, - options: const StorageGetPropertiesOptions(), + options: StorageGetPropertiesOptions(bucket: options.bucket), )) .storageItem; @@ -71,6 +71,7 @@ Future _downloadFromUrl({ pluginOptions: S3GetUrlPluginOptions( useAccelerateEndpoint: s3PluginOptions.useAccelerateEndpoint, ), + bucket: options.bucket, ), )) .url; diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/platform_impl/download_file/download_file_io.dart b/packages/storage/amplify_storage_s3_dart/lib/src/platform_impl/download_file/download_file_io.dart index 5651870acd..c17e19b1db 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/platform_impl/download_file/download_file_io.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/platform_impl/download_file/download_file_io.dart @@ -32,6 +32,7 @@ S3DownloadFileOperation downloadFile({ getProperties: s3PluginOptions.getProperties, useAccelerateEndpoint: s3PluginOptions.useAccelerateEndpoint, ), + bucket: options.bucket, ); final downloadDataTask = storageS3Service.downloadData( From 7a46411597105225477db5eaaab8c983d6a47ed6 Mon Sep 17 00:00:00 2001 From: Tyler-Larkin Date: Tue, 19 Nov 2024 15:41:11 -0800 Subject: [PATCH 23/25] feat(storage): multi bucket copy (#5674) * chore(core): Added toJson to storage option buckets * feat(storage): multi bucket copy --- .../lib/src/types/storage/bucket_info.dart | 9 +- .../lib/src/types/storage/copy_buckets.dart | 24 ++++++ .../lib/src/types/storage/copy_options.dart | 13 ++- .../types/storage/download_data_options.dart | 2 +- .../types/storage/download_file_options.dart | 2 +- .../types/storage/get_properties_options.dart | 2 +- .../src/types/storage/get_url_options.dart | 2 +- .../lib/src/types/storage/list_options.dart | 2 +- .../types/storage/remove_many_options.dart | 2 +- .../lib/src/types/storage/remove_options.dart | 2 +- .../lib/src/types/storage/storage_bucket.dart | 9 +- .../storage/storage_bucket_from_outputs.dart | 5 ++ .../lib/src/types/storage/storage_types.dart | 1 + .../types/storage/upload_data_options.dart | 2 +- .../types/storage/upload_file_options.dart | 2 +- .../example/integration_test/copy_test.dart | 85 +++++++++++++++++++ .../lib/src/amplify_storage_s3_dart_impl.dart | 1 + .../service/storage_s3_service_impl.dart | 14 +-- 18 files changed, 159 insertions(+), 20 deletions(-) create mode 100644 packages/amplify_core/lib/src/types/storage/copy_buckets.dart diff --git a/packages/amplify_core/lib/src/types/storage/bucket_info.dart b/packages/amplify_core/lib/src/types/storage/bucket_info.dart index a30811b333..811871f9b3 100644 --- a/packages/amplify_core/lib/src/types/storage/bucket_info.dart +++ b/packages/amplify_core/lib/src/types/storage/bucket_info.dart @@ -3,7 +3,8 @@ import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.bucket_info} /// Presents a storage bucket information. /// {@endtemplate} -class BucketInfo with AWSEquatable { +class BucketInfo + with AWSEquatable, AWSSerializable> { /// {@macro amplify_core.storage.bucket_info} const BucketInfo({required this.bucketName, required this.region}); final String bucketName; @@ -14,4 +15,10 @@ class BucketInfo with AWSEquatable { bucketName, region, ]; + + @override + Map toJson() => { + 'bucketName': bucketName, + 'region': region, + }; } diff --git a/packages/amplify_core/lib/src/types/storage/copy_buckets.dart b/packages/amplify_core/lib/src/types/storage/copy_buckets.dart new file mode 100644 index 0000000000..7bfb83575d --- /dev/null +++ b/packages/amplify_core/lib/src/types/storage/copy_buckets.dart @@ -0,0 +1,24 @@ +import 'package:amplify_core/amplify_core.dart'; + +/// Presents storage buckets for a copy operation. +class CopyBuckets with AWSSerializable> { + /// Creates a [CopyBuckets] with [source] and [destination] buckets. + const CopyBuckets({ + required this.source, + required this.destination, + }); + + /// Creates a [CopyBuckets] with the same [bucket] for the [source] and [destination]. + CopyBuckets.sameBucket(StorageBucket bucket) + : source = bucket, + destination = bucket; + + final StorageBucket source; + final StorageBucket destination; + + @override + Map toJson() => { + 'source': source.toJson(), + 'destination': destination.toJson(), + }; +} diff --git a/packages/amplify_core/lib/src/types/storage/copy_options.dart b/packages/amplify_core/lib/src/types/storage/copy_options.dart index eacd7f4ff1..a9910ecfff 100644 --- a/packages/amplify_core/lib/src/types/storage/copy_options.dart +++ b/packages/amplify_core/lib/src/types/storage/copy_options.dart @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import 'package:aws_common/aws_common.dart'; +import 'package:amplify_core/amplify_core.dart'; /// {@template amplify_core.storage.copy_options} /// Configurable options for `Amplify.Storage.copy`. @@ -12,13 +12,19 @@ class StorageCopyOptions AWSSerializable>, AWSDebuggable { /// {@macro amplify_core.storage.copy_options} - const StorageCopyOptions({this.pluginOptions}); + const StorageCopyOptions({ + this.pluginOptions, + this.buckets, + }); /// plugin specific options for `Amplify.Storage.copy`. final StorageCopyPluginOptions? pluginOptions; + /// Optionally specify which buckets to target + final CopyBuckets? buckets; + @override - List get props => [pluginOptions]; + List get props => [pluginOptions, buckets]; @override String get runtimeTypeName => 'StorageCopyOptions'; @@ -26,6 +32,7 @@ class StorageCopyOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), + 'buckets': buckets?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/download_data_options.dart b/packages/amplify_core/lib/src/types/storage/download_data_options.dart index 020856dfe1..26a4ea2422 100644 --- a/packages/amplify_core/lib/src/types/storage/download_data_options.dart +++ b/packages/amplify_core/lib/src/types/storage/download_data_options.dart @@ -32,7 +32,7 @@ class StorageDownloadDataOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/download_file_options.dart b/packages/amplify_core/lib/src/types/storage/download_file_options.dart index 23d745f5e6..69be1758cd 100644 --- a/packages/amplify_core/lib/src/types/storage/download_file_options.dart +++ b/packages/amplify_core/lib/src/types/storage/download_file_options.dart @@ -32,7 +32,7 @@ class StorageDownloadFileOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/get_properties_options.dart b/packages/amplify_core/lib/src/types/storage/get_properties_options.dart index dbc5b8c112..9a2216b40d 100644 --- a/packages/amplify_core/lib/src/types/storage/get_properties_options.dart +++ b/packages/amplify_core/lib/src/types/storage/get_properties_options.dart @@ -32,7 +32,7 @@ class StorageGetPropertiesOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/get_url_options.dart b/packages/amplify_core/lib/src/types/storage/get_url_options.dart index 62954e8281..161cc2b93b 100644 --- a/packages/amplify_core/lib/src/types/storage/get_url_options.dart +++ b/packages/amplify_core/lib/src/types/storage/get_url_options.dart @@ -35,7 +35,7 @@ class StorageGetUrlOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/list_options.dart b/packages/amplify_core/lib/src/types/storage/list_options.dart index f0a1d80962..72570f262b 100644 --- a/packages/amplify_core/lib/src/types/storage/list_options.dart +++ b/packages/amplify_core/lib/src/types/storage/list_options.dart @@ -41,7 +41,7 @@ class StorageListOptions Map toJson() => { 'pageSize': pageSize, 'nextToken': nextToken, - 'bucket': bucket, + 'bucket': bucket?.toJson(), 'pluginOptions': pluginOptions?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/remove_many_options.dart b/packages/amplify_core/lib/src/types/storage/remove_many_options.dart index 05db2da027..a50c651fc8 100644 --- a/packages/amplify_core/lib/src/types/storage/remove_many_options.dart +++ b/packages/amplify_core/lib/src/types/storage/remove_many_options.dart @@ -35,7 +35,7 @@ class StorageRemoveManyOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/remove_options.dart b/packages/amplify_core/lib/src/types/storage/remove_options.dart index 95be578c1c..13e68cb501 100644 --- a/packages/amplify_core/lib/src/types/storage/remove_options.dart +++ b/packages/amplify_core/lib/src/types/storage/remove_options.dart @@ -32,7 +32,7 @@ class StorageRemoveOptions @override Map toJson() => { 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/storage_bucket.dart b/packages/amplify_core/lib/src/types/storage/storage_bucket.dart index 711e30d18b..ccfdd345a9 100644 --- a/packages/amplify_core/lib/src/types/storage/storage_bucket.dart +++ b/packages/amplify_core/lib/src/types/storage/storage_bucket.dart @@ -1,10 +1,10 @@ +import 'package:amplify_core/amplify_core.dart'; import 'package:amplify_core/src/config/amplify_outputs/storage/storage_outputs.dart'; -import 'package:amplify_core/src/types/storage/bucket_info.dart'; import 'package:amplify_core/src/types/storage/storage_bucket_from_outputs.dart'; import 'package:meta/meta.dart'; /// Presents a storage bucket. -class StorageBucket { +class StorageBucket with AWSSerializable> { /// Creates a [StorageBucket] from [BucketInfo]. const StorageBucket.fromBucketInfo(this._info); @@ -16,4 +16,9 @@ class StorageBucket { @internal BucketInfo resolveBucketInfo(StorageOutputs? storageOutputs) => _info; + + @override + Map toJson() => { + '_info': _info.toJson(), + }; } diff --git a/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart b/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart index ed22c24aa2..b51d17029d 100644 --- a/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart +++ b/packages/amplify_core/lib/src/types/storage/storage_bucket_from_outputs.dart @@ -40,4 +40,9 @@ class StorageBucketFromOutputs implements StorageBucket { region: bucket.awsRegion, ); } + + @override + Map toJson() => { + '_name': _name, + }; } diff --git a/packages/amplify_core/lib/src/types/storage/storage_types.dart b/packages/amplify_core/lib/src/types/storage/storage_types.dart index 1e4bfbfb02..b00b66a362 100644 --- a/packages/amplify_core/lib/src/types/storage/storage_types.dart +++ b/packages/amplify_core/lib/src/types/storage/storage_types.dart @@ -12,6 +12,7 @@ export '../exception/amplify_exception.dart' NetworkException, UnknownException; export 'bucket_info.dart'; +export 'copy_buckets.dart'; export 'copy_operation.dart'; export 'copy_options.dart'; export 'copy_request.dart'; diff --git a/packages/amplify_core/lib/src/types/storage/upload_data_options.dart b/packages/amplify_core/lib/src/types/storage/upload_data_options.dart index 3b7bd9eb7b..83e8f5009c 100644 --- a/packages/amplify_core/lib/src/types/storage/upload_data_options.dart +++ b/packages/amplify_core/lib/src/types/storage/upload_data_options.dart @@ -37,7 +37,7 @@ class StorageUploadDataOptions Map toJson() => { 'metadata': metadata, 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/amplify_core/lib/src/types/storage/upload_file_options.dart b/packages/amplify_core/lib/src/types/storage/upload_file_options.dart index 6b025257fb..2422eef043 100644 --- a/packages/amplify_core/lib/src/types/storage/upload_file_options.dart +++ b/packages/amplify_core/lib/src/types/storage/upload_file_options.dart @@ -37,7 +37,7 @@ class StorageUploadFileOptions Map toJson() => { 'metadata': metadata, 'pluginOptions': pluginOptions?.toJson(), - 'bucket': bucket, + 'bucket': bucket?.toJson(), }; } diff --git a/packages/storage/amplify_storage_s3/example/integration_test/copy_test.dart b/packages/storage/amplify_storage_s3/example/integration_test/copy_test.dart index 87afda83b3..12e892ea06 100644 --- a/packages/storage/amplify_storage_s3/example/integration_test/copy_test.dart +++ b/packages/storage/amplify_storage_s3/example/integration_test/copy_test.dart @@ -138,5 +138,90 @@ void main() { expect(result.copiedItem.path, destinationPath); }); }); + + group('multi bucket', () { + final data = 'copy data'.codeUnits; + final bucket1 = StorageBucket.fromOutputs( + 'Storage Integ Test main bucket', + ); + final bucket2 = StorageBucket.fromOutputs( + 'Storage Integ Test secondary bucket', + ); + final bucket1PathSource = 'public/multi-bucket-get-url-${uuid()}'; + final bucket2PathSource = 'public/multi-bucket-get-url-${uuid()}'; + final bucket2PathDestination = 'public/multi-bucket-get-url-${uuid()}'; + final storageBucket1PathSource = + StoragePath.fromString(bucket1PathSource); + final storageBucket2PathSource = + StoragePath.fromString(bucket2PathSource); + final storageBucket2PathDestination = + StoragePath.fromString(bucket2PathDestination); + + setUp(() async { + await configure(amplifyEnvironments['main']!); + addTearDownPath(storageBucket1PathSource); + addTearDownPath(storageBucket2PathSource); + addTearDownPath(storageBucket2PathDestination); + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes(data), + path: storageBucket1PathSource, + options: StorageUploadDataOptions( + bucket: bucket1, + ), + ).result; + await Amplify.Storage.uploadData( + data: StorageDataPayload.bytes(data), + path: storageBucket2PathSource, + options: StorageUploadDataOptions( + bucket: bucket2, + ), + ).result; + }); + + testWidgets('copy to a different bucket', (_) async { + final result = await Amplify.Storage.copy( + source: storageBucket1PathSource, + destination: storageBucket2PathDestination, + options: StorageCopyOptions( + buckets: CopyBuckets( + source: bucket1, + destination: bucket2, + ), + ), + ).result; + expect(result.copiedItem.path, bucket2PathDestination); + + final downloadResult = await Amplify.Storage.downloadData( + path: storageBucket2PathDestination, + options: StorageDownloadDataOptions(bucket: bucket2), + ).result; + expect( + downloadResult.bytes, + data, + ); + }); + + testWidgets('copy to the same bucket', (_) async { + final result = await Amplify.Storage.copy( + source: storageBucket2PathSource, + destination: storageBucket2PathDestination, + options: StorageCopyOptions( + buckets: CopyBuckets.sameBucket( + bucket2, + ), + ), + ).result; + expect(result.copiedItem.path, bucket2PathDestination); + + final downloadResult = await Amplify.Storage.downloadData( + path: storageBucket2PathDestination, + options: StorageDownloadDataOptions(bucket: bucket2), + ).result; + expect( + downloadResult.bytes, + data, + ); + }); + }); }); } diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart index f754941b43..11a2c9bc9c 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/amplify_storage_s3_dart_impl.dart @@ -364,6 +364,7 @@ class AmplifyStorageS3Dart extends StoragePluginInterface final s3Options = StorageCopyOptions( pluginOptions: s3PluginOptions, + buckets: options?.buckets, ); return S3CopyOperation( diff --git a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart index 05b71ea05f..d3d8939f39 100644 --- a/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart +++ b/packages/storage/amplify_storage_s3_dart/lib/src/storage_s3_service/service/storage_s3_service_impl.dart @@ -412,6 +412,10 @@ class StorageS3Service { }) async { final s3PluginOptions = options.pluginOptions as S3CopyPluginOptions? ?? const S3CopyPluginOptions(); + final s3ClientInfoSource = + getS3ClientInfo(storageBucket: options.buckets?.source); + final s3ClientInfoDestination = + getS3ClientInfo(storageBucket: options.buckets?.destination); final [sourcePath, destinationPath] = await _pathResolver.resolvePaths( paths: [source, destination], @@ -419,14 +423,14 @@ class StorageS3Service { final copyRequest = s3.CopyObjectRequest.build((builder) { builder - ..bucket = _storageOutputs.bucketName - ..copySource = '${_storageOutputs.bucketName}/$sourcePath' + ..bucket = s3ClientInfoDestination.bucketName + ..copySource = '${s3ClientInfoSource.bucketName}/$sourcePath' ..key = destinationPath ..metadataDirective = s3.MetadataDirective.copy; }); try { - await _defaultS3Client.copyObject(copyRequest).result; + await s3ClientInfoDestination.client.copyObject(copyRequest).result; } on smithy.UnknownSmithyHttpException catch (error) { // S3Client.copyObject may return 403 or 404 error throw error.toStorageException(); @@ -438,8 +442,8 @@ class StorageS3Service { copiedItem: s3PluginOptions.getProperties ? S3Item.fromHeadObjectOutput( await headObject( - s3client: _defaultS3Client, - bucket: _storageOutputs.bucketName, + s3client: s3ClientInfoDestination.client, + bucket: s3ClientInfoDestination.bucketName, key: destinationPath, ), path: destinationPath, From 0db2d4fd2cf340e2b2cdf5f6ce05cb87401995e6 Mon Sep 17 00:00:00 2001 From: NikaHsn Date: Tue, 26 Nov 2024 11:30:12 -0800 Subject: [PATCH 24/25] chore(storage): test data to use clear random names for bucket info (#5724) --- .../config/amplify_outputs/test_data.dart | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/amplify_core/test/config/amplify_outputs/test_data.dart b/packages/amplify_core/test/config/amplify_outputs/test_data.dart index 4a1f5e7b87..887e46a260 100644 --- a/packages/amplify_core/test/config/amplify_outputs/test_data.dart +++ b/packages/amplify_core/test/config/amplify_outputs/test_data.dart @@ -5,8 +5,8 @@ // It uses json-schema-faker to generate a sample json from the Amplify GEN 2 client-config-schema. // Run below commands to regenerate a sample json. -// curl https://raw.githubusercontent.com/aws-amplify/amplify-backend/main/packages/client-config/src/client-config-schema/schema_v1.json -o schema_v1.json -// npx json-schema-faker -s schema_v1.json -o sample.json --alwaysFakeOptionals +// curl https://raw.githubusercontent.com/aws-amplify/amplify-backend/main/packages/client-config/src/client-config-schema/schema_v1.3.json -o schema_v1.3.json +// npx json-schema-faker -s schema_v1.3.json -o sample.json --alwaysFakeOptionals const amplifyoutputs = '''{ "schema": "dolor nisi incididunt adipisicing", @@ -99,24 +99,24 @@ const amplifyoutputs = '''{ ] }, "storage": { - "aws_region": "us-east-2", - "bucket_name": "amplifyTeamDrive-one-stora-testbucketgen2bucket0b8c-9ggcfqfunkjr", + "aws_region": "mollit culpa non dolore sint", + "bucket_name": "incididunt minim nulla", "buckets": [ - { - "name": "amplifyTeamDrive-one", - "bucket_name": "amplifyTeamDrive-one-stora-testbucketgen2bucket0b8c-9ggcfqfunkjr", - "aws_region": "us-east-2" - }, - { - "name": "amplifyTeamDrive-two", - "bucket_name": "amplifyTeamDrive-two-stora-testbucketgen2bucket0b8c-2", - "aws_region": "us-east-2" - }, - { - "name": "amplifyTeamDrive-three", - "bucket_name": "amplifyTeamDrive-three-stora-testbucketgen2bucket0b8c-3", - "aws_region": "us-east-2" - } + { + "aws_region": "mollit culpa non dolore sint", + "bucket_name": "incididunt minim nulla", + "name": "ullamco consectetur dolore" + }, + { + "aws_region": "Duis commodo", + "bucket_name": "sint", + "name": "ex non" + }, + { + "aws_region": "enim cillum eiusmod", + "bucket_name": "proident ullamco deserunt", + "name": "minim elit" + } ] }, "version": "1" From 4622de3619b295efe25cd4d205c72edcc7c8fe5f Mon Sep 17 00:00:00 2001 From: Nika Hassani Date: Tue, 3 Dec 2024 14:32:10 -0800 Subject: [PATCH 25/25] chore(ci): rebase with main to fix the failing tests --- .../test/storage_s3_service/task/s3_upload_task_test.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/task/s3_upload_task_test.dart b/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/task/s3_upload_task_test.dart index 7da7824619..85e96ec5d9 100644 --- a/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/task/s3_upload_task_test.dart +++ b/packages/storage/amplify_storage_s3_dart/test/storage_s3_service/task/s3_upload_task_test.dart @@ -1337,9 +1337,10 @@ void main() { final uploadTask = S3UploadTask.fromAWSFile( mockFile, s3Client: s3Client, - defaultS3ClientConfig: defaultS3ClientConfig, + s3ClientConfig: defaultS3ClientConfig, pathResolver: pathResolver, bucket: testBucket, + awsRegion: testRegion, path: const StoragePath.fromString(testKey), options: testUploadDataOptions, logger: logger,