From 912798e6ab473df24ae5001c21573ed58412459f Mon Sep 17 00:00:00 2001 From: Ramon Date: Fri, 24 Nov 2023 14:55:26 +0100 Subject: [PATCH 1/2] update cloudformation_actions.py --- pycfmodel/cloudformation_actions.py | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/pycfmodel/cloudformation_actions.py b/pycfmodel/cloudformation_actions.py index 63bdca3..6f312e0 100644 --- a/pycfmodel/cloudformation_actions.py +++ b/pycfmodel/cloudformation_actions.py @@ -2080,6 +2080,12 @@ "cloudformation:UpdateStackSet", "cloudformation:UpdateTerminationProtection", "cloudformation:ValidateTemplate", + "cloudfront-keyvaluestore:DeleteKey", + "cloudfront-keyvaluestore:DescribeKeyValueStore", + "cloudfront-keyvaluestore:GetKey", + "cloudfront-keyvaluestore:ListKeys", + "cloudfront-keyvaluestore:PutKey", + "cloudfront-keyvaluestore:UpdateKeys", "cloudfront:AssociateAlias", "cloudfront:CopyDistribution", "cloudfront:CreateCachePolicy", @@ -4536,6 +4542,7 @@ "ec2:AssociateEnclaveCertificateIamRole", "ec2:AssociateIamInstanceProfile", "ec2:AssociateInstanceEventWindow", + "ec2:AssociateIpamByoasn", "ec2:AssociateIpamResourceDiscovery", "ec2:AssociateNatGatewayAddress", "ec2:AssociateRouteTable", @@ -4732,6 +4739,7 @@ "ec2:DeleteVpnConnectionRoute", "ec2:DeleteVpnGateway", "ec2:DeprovisionByoipCidr", + "ec2:DeprovisionIpamByoasn", "ec2:DeprovisionIpamPoolCidr", "ec2:DeprovisionPublicIpv4PoolCidr", "ec2:DeregisterImage", @@ -4747,6 +4755,7 @@ "ec2:DescribeAwsNetworkPerformanceMetricSubscriptions", "ec2:DescribeBundleTasks", "ec2:DescribeByoipCidrs", + "ec2:DescribeCapacityBlockOfferings", "ec2:DescribeCapacityReservationFleets", "ec2:DescribeCapacityReservations", "ec2:DescribeCarrierGateways", @@ -4788,10 +4797,12 @@ "ec2:DescribeInstanceEventNotificationAttributes", "ec2:DescribeInstanceEventWindows", "ec2:DescribeInstanceStatus", + "ec2:DescribeInstanceTopology", "ec2:DescribeInstanceTypeOfferings", "ec2:DescribeInstanceTypes", "ec2:DescribeInstances", "ec2:DescribeInternetGateways", + "ec2:DescribeIpamByoasn", "ec2:DescribeIpamPools", "ec2:DescribeIpamResourceDiscoveries", "ec2:DescribeIpamResourceDiscoveryAssociations", @@ -4808,6 +4819,7 @@ "ec2:DescribeLocalGatewayVirtualInterfaceGroups", "ec2:DescribeLocalGatewayVirtualInterfaces", "ec2:DescribeLocalGateways", + "ec2:DescribeLockedSnapshots", "ec2:DescribeManagedPrefixLists", "ec2:DescribeMovingAddresses", "ec2:DescribeNatGateways", @@ -4901,6 +4913,7 @@ "ec2:DisableImageDeprecation", "ec2:DisableIpamOrganizationAdminAccount", "ec2:DisableSerialConsoleAccess", + "ec2:DisableSnapshotBlockPublicAccess", "ec2:DisableTransitGatewayRouteTablePropagation", "ec2:DisableVgwRoutePropagation", "ec2:DisableVpcClassicLink", @@ -4910,6 +4923,7 @@ "ec2:DisassociateEnclaveCertificateIamRole", "ec2:DisassociateIamInstanceProfile", "ec2:DisassociateInstanceEventWindow", + "ec2:DisassociateIpamByoasn", "ec2:DisassociateIpamResourceDiscovery", "ec2:DisassociateNatGatewayAddress", "ec2:DisassociateRouteTable", @@ -4931,6 +4945,7 @@ "ec2:EnableIpamOrganizationAdminAccount", "ec2:EnableReachabilityAnalyzerOrganizationSharing", "ec2:EnableSerialConsoleAccess", + "ec2:EnableSnapshotBlockPublicAccess", "ec2:EnableTransitGatewayRouteTablePropagation", "ec2:EnableVgwRoutePropagation", "ec2:EnableVolumeIO", @@ -4958,6 +4973,7 @@ "ec2:GetInstanceUefiData", "ec2:GetIpamAddressHistory", "ec2:GetIpamDiscoveredAccounts", + "ec2:GetIpamDiscoveredPublicAddresses", "ec2:GetIpamDiscoveredResourceCidrs", "ec2:GetIpamPoolAllocations", "ec2:GetIpamPoolCidrs", @@ -4972,6 +4988,7 @@ "ec2:GetResourcePolicy", "ec2:GetSecurityGroupsForVpc", "ec2:GetSerialConsoleAccessStatus", + "ec2:GetSnapshotBlockPublicAccessState", "ec2:GetSpotPlacementScores", "ec2:GetSubnetCidrReservations", "ec2:GetTransitGatewayAttachmentPropagations", @@ -4997,6 +5014,7 @@ "ec2:InjectApiError", "ec2:ListImagesInRecycleBin", "ec2:ListSnapshotsInRecycleBin", + "ec2:LockSnapshot", "ec2:ModifyAddressAttribute", "ec2:ModifyAvailabilityZoneGroup", "ec2:ModifyCapacityReservation", @@ -5066,8 +5084,10 @@ "ec2:MoveByoipCidrToIpam", "ec2:PauseVolumeIO", "ec2:ProvisionByoipCidr", + "ec2:ProvisionIpamByoasn", "ec2:ProvisionIpamPoolCidr", "ec2:ProvisionPublicIpv4PoolCidr", + "ec2:PurchaseCapacityBlock", "ec2:PurchaseHostReservation", "ec2:PurchaseReservedInstancesOffering", "ec2:PurchaseScheduledInstances", @@ -5127,6 +5147,7 @@ "ec2:UnassignIpv6Addresses", "ec2:UnassignPrivateIpAddresses", "ec2:UnassignPrivateNatGatewayAddress", + "ec2:UnlockSnapshot", "ec2:UnmonitorInstances", "ec2:UpdateSecurityGroupRuleDescriptionsEgress", "ec2:UpdateSecurityGroupRuleDescriptionsIngress", @@ -7859,6 +7880,7 @@ "iotsitewise:CreateAccessPolicy", "iotsitewise:CreateAsset", "iotsitewise:CreateAssetModel", + "iotsitewise:CreateAssetModelCompositeModel", "iotsitewise:CreateBulkImportJob", "iotsitewise:CreateDashboard", "iotsitewise:CreateGateway", @@ -7867,14 +7889,18 @@ "iotsitewise:DeleteAccessPolicy", "iotsitewise:DeleteAsset", "iotsitewise:DeleteAssetModel", + "iotsitewise:DeleteAssetModelCompositeModel", "iotsitewise:DeleteDashboard", "iotsitewise:DeleteGateway", "iotsitewise:DeletePortal", "iotsitewise:DeleteProject", "iotsitewise:DeleteTimeSeries", "iotsitewise:DescribeAccessPolicy", + "iotsitewise:DescribeAction", "iotsitewise:DescribeAsset", + "iotsitewise:DescribeAssetCompositeModel", "iotsitewise:DescribeAssetModel", + "iotsitewise:DescribeAssetModelCompositeModel", "iotsitewise:DescribeAssetProperty", "iotsitewise:DescribeBulkImportJob", "iotsitewise:DescribeDashboard", @@ -7888,11 +7914,16 @@ "iotsitewise:DescribeTimeSeries", "iotsitewise:DisassociateAssets", "iotsitewise:DisassociateTimeSeriesFromAssetProperty", + "iotsitewise:EnableSiteWiseIntegration", + "iotsitewise:ExecuteAction", + "iotsitewise:ExecuteQuery", "iotsitewise:GetAssetPropertyAggregates", "iotsitewise:GetAssetPropertyValue", "iotsitewise:GetAssetPropertyValueHistory", "iotsitewise:GetInterpolatedAssetPropertyValues", "iotsitewise:ListAccessPolicies", + "iotsitewise:ListActions", + "iotsitewise:ListAssetModelCompositeModels", "iotsitewise:ListAssetModelProperties", "iotsitewise:ListAssetModels", "iotsitewise:ListAssetProperties", @@ -7900,6 +7931,7 @@ "iotsitewise:ListAssets", "iotsitewise:ListAssociatedAssets", "iotsitewise:ListBulkImportJobs", + "iotsitewise:ListCompositionRelationships", "iotsitewise:ListDashboards", "iotsitewise:ListGateways", "iotsitewise:ListPortals", @@ -7915,6 +7947,7 @@ "iotsitewise:UpdateAccessPolicy", "iotsitewise:UpdateAsset", "iotsitewise:UpdateAssetModel", + "iotsitewise:UpdateAssetModelCompositeModel", "iotsitewise:UpdateAssetModelPropertyRouting", "iotsitewise:UpdateAssetProperty", "iotsitewise:UpdateDashboard", @@ -7923,8 +7956,10 @@ "iotsitewise:UpdatePortal", "iotsitewise:UpdateProject", "iottwinmaker:BatchPutPropertyValues", + "iottwinmaker:CancelMetadataTransferJob", "iottwinmaker:CreateComponentType", "iottwinmaker:CreateEntity", + "iottwinmaker:CreateMetadataTransferJob", "iottwinmaker:CreateScene", "iottwinmaker:CreateSyncJob", "iottwinmaker:CreateWorkspace", @@ -7936,6 +7971,7 @@ "iottwinmaker:ExecuteQuery", "iottwinmaker:GetComponentType", "iottwinmaker:GetEntity", + "iottwinmaker:GetMetadataTransferJob", "iottwinmaker:GetPricingPlan", "iottwinmaker:GetPropertyValue", "iottwinmaker:GetPropertyValueHistory", @@ -7943,7 +7979,10 @@ "iottwinmaker:GetSyncJob", "iottwinmaker:GetWorkspace", "iottwinmaker:ListComponentTypes", + "iottwinmaker:ListComponents", "iottwinmaker:ListEntities", + "iottwinmaker:ListMetadataTransferJobs", + "iottwinmaker:ListProperties", "iottwinmaker:ListScenes", "iottwinmaker:ListSyncJobs", "iottwinmaker:ListSyncResources", @@ -8370,6 +8409,7 @@ "kinesis:AddTagsToStream", "kinesis:CreateStream", "kinesis:DecreaseStreamRetentionPeriod", + "kinesis:DeleteResourcePolicy", "kinesis:DeleteStream", "kinesis:DeregisterStreamConsumer", "kinesis:DescribeLimits", @@ -8379,6 +8419,7 @@ "kinesis:DisableEnhancedMonitoring", "kinesis:EnableEnhancedMonitoring", "kinesis:GetRecords", + "kinesis:GetResourcePolicy", "kinesis:GetShardIterator", "kinesis:IncreaseStreamRetentionPeriod", "kinesis:ListShards", @@ -8388,6 +8429,7 @@ "kinesis:MergeShards", "kinesis:PutRecord", "kinesis:PutRecords", + "kinesis:PutResourcePolicy", "kinesis:RegisterStreamConsumer", "kinesis:RemoveTagsFromStream", "kinesis:SplitShard", @@ -11574,6 +11616,7 @@ "redshift:CreateEventSubscription", "redshift:CreateHsmClientCertificate", "redshift:CreateHsmConfiguration", + "redshift:CreateRedshiftIdcApplication", "redshift:CreateSavedQuery", "redshift:CreateScheduledAction", "redshift:CreateSnapshotCopyGrant", @@ -11593,6 +11636,7 @@ "redshift:DeleteHsmClientCertificate", "redshift:DeleteHsmConfiguration", "redshift:DeletePartner", + "redshift:DeleteRedshiftIdcApplication", "redshift:DeleteResourcePolicy", "redshift:DeleteSavedQueries", "redshift:DeleteScheduledAction", @@ -11629,6 +11673,7 @@ "redshift:DescribeOrderableClusterOptions", "redshift:DescribePartners", "redshift:DescribeQuery", + "redshift:DescribeRedshiftIdcApplications", "redshift:DescribeReservedNodeExchangeStatus", "redshift:DescribeReservedNodeOfferings", "redshift:DescribeReservedNodes", @@ -11673,6 +11718,7 @@ "redshift:ModifyCustomDomainAssociation", "redshift:ModifyEndpointAccess", "redshift:ModifyEventSubscription", + "redshift:ModifyRedshiftIdcApplication", "redshift:ModifySavedQuery", "redshift:ModifyScheduledAction", "redshift:ModifySnapshotCopyRetentionPeriod", From 1cb12f3a3a449556a66770c284974d52c572784c Mon Sep 17 00:00:00 2001 From: Ramon Date: Fri, 24 Nov 2023 14:56:01 +0100 Subject: [PATCH 2/2] add globals to model and able to resolve it --- pycfmodel/model/cf_model.py | 25 +++++++++++++-- tests/test_cf_model.py | 62 +++++++++++++++++++++++++++++++++++++ tests/test_resolver.py | 22 +++++++++++++ 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/pycfmodel/model/cf_model.py b/pycfmodel/model/cf_model.py index 19c5951..734aca0 100644 --- a/pycfmodel/model/cf_model.py +++ b/pycfmodel/model/cf_model.py @@ -18,9 +18,10 @@ class CFModel(CustomModel): Properties: - - AWSTemplateFormatVersion + - AWSTemplateFormatVersion: The AWS CloudFormation template version that the template conforms to. - Conditions: Conditions that control behaviour of the template. - Description: Description for the template. + - Globals: TODO - Mappings: A 3 level mapping of keys and associated values. - Metadata: Additional information about the template. - Outputs: Output values of the template. @@ -30,11 +31,14 @@ class CFModel(CustomModel): - Transform: For serverless applications, specifies the version of the AWS Serverless Application Model (AWS SAM) to use. More info at [AWS Docs](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-anatomy.html) + + More info for Globals at [AWS Globals Docs](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-specification-template-anatomy-globals.html) """ AWSTemplateFormatVersion: Optional[date] Conditions: Optional[Dict] = {} Description: Optional[str] = None + Globals: Optional[Dict] = {} Mappings: Optional[Dict[str, Dict[str, Dict[str, Any]]]] = {} Metadata: Optional[Dict[str, Any]] = None Outputs: Optional[Dict[str, Dict[str, Union[str, Dict]]]] = {} @@ -84,6 +88,12 @@ def resolve(self, extra_params=None) -> "CFModel": {key: _extended_bool(resolve(value, extended_parameters, self.Mappings, resolved_conditions))} ) + globals_cf = dict_value.pop("Globals", {}) + resolved_globals = { + key: resolve(value, extended_parameters, self.Mappings, resolved_conditions) + for key, value in globals_cf.items() + } + resources = dict_value.pop("Resources") resolved_resources = { key: resolve(value, extended_parameters, self.Mappings, resolved_conditions) @@ -91,7 +101,10 @@ def resolve(self, extra_params=None) -> "CFModel": if value.get("Condition") is None or (value.get("Condition") is not None and resolved_conditions.get(value["Condition"], True)) } - return CFModel(**dict_value, Conditions=resolved_conditions, Resources=resolved_resources) + + return CFModel( + **dict_value, Conditions=resolved_conditions, Resources=resolved_resources, Globals=resolved_globals + ) def expand_actions(self) -> "CFModel": """ @@ -134,3 +147,11 @@ def resources_filtered_by_type( if isinstance(resource, allowed_resource_classes) or resource.Type in allowed_types: result[resource_name] = resource return result + + def is_sam_model(self) -> bool: + transform = self.Transform + if self.Transform is None: + return False + if isinstance(transform, str): + transform = [transform] + return any(macro.startswith("AWS::Serverless") for macro in transform) diff --git a/tests/test_cf_model.py b/tests/test_cf_model.py index acccc85..32a11f4 100644 --- a/tests/test_cf_model.py +++ b/tests/test_cf_model.py @@ -18,6 +18,55 @@ def model(): "Resources": {"Logical ID": {"Type": "Resource type", "Properties": {"foo": "bar"}}}, "Rules": {}, "Outputs": {}, + "Globals": {}, + } + ) + + +@pytest.fixture() +def model_single_transform(): + return CFModel( + **{ + "AWSTemplateFormatVersion": "2012-12-12", + "Description": "", + "Metadata": {}, + "Parameters": {}, + "Mappings": {}, + "Conditions": {}, + "Transform": "AWS::Serverless-2016-10-31", + "Resources": {"Logical ID": {"Type": "Resource type", "Properties": {"foo": "bar"}}}, + "Rules": {}, + "Outputs": {}, + } + ) + + +@pytest.fixture() +def model_no_transform(): + return CFModel( + **{ + "AWSTemplateFormatVersion": "2012-12-12", + "Description": "", + "Metadata": {}, + "Parameters": {}, + "Mappings": {}, + "Conditions": {}, + "Resources": {"Logical ID": {"Type": "Resource type", "Properties": {"foo": "bar"}}}, + "Rules": {}, + "Outputs": {}, + } + ) + + +@pytest.fixture() +def model_with_empty_globals(): + return CFModel( + **{ + "AWSTemplateFormatVersion": "2012-12-12", + "Globals": {}, + "Parameters": {}, + "Transform": "AWS::Serverless-2016-10-31", + "Resources": {"Logical ID": {"Type": "AWS::Dummy::Dummy", "Properties": {"foo": "bar"}}}, } ) @@ -26,6 +75,7 @@ def test_basic_json(model: CFModel): assert type(model).__name__ == "CFModel" assert len(model.Resources) == 1 assert model.Transform == ["MyMacro", "AWS::Serverless"] + assert model.Globals == {} def test_resources_filtered_by_type(): @@ -61,3 +111,15 @@ def test_transform_handles_string(): def test_resolve_model(model): assert model.resolve() == model + + +@pytest.mark.parametrize( + "model_fixture,is_sam_model", + [("model", True), ("model_single_transform", True), ("model_no_transform", False)], +) +def test_transform_is_of_type_sam_model(model_fixture, is_sam_model, request): + assert request.getfixturevalue(model_fixture).is_sam_model() is is_sam_model + + +def test_model_with_empty_globals_is_able_to_resolve_to_empty_dict(model_with_empty_globals): + assert model_with_empty_globals.Globals == {} diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 34f4be2..db6ad50 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -815,3 +815,25 @@ def test_resolve_find_in_map_for_bool_values_in_map(params, expected_resolved_va result = resolve_find_in_map(function_body=function_body, params=params, mappings=mappings, conditions={}) assert result == expected_resolved_value + + +def test_resolve_globals_with_values_and_referencing_parameters(): + """ + This test aims to be able to solve these type of templates: + https://github.com/Skyscanner/cfripper/issues/259#issuecomment-1824485673 + """ + template = { + "AWSTemplateFormatVersion": "2012-12-12", + "Transform": "AWS::Serverless-2016-10-31", + "Parameters": { + "ProjectName": {"Type": "String", "Default": "my-project"}, + "Environment": {"Type": "String", "Default": "development"}, + }, + "Globals": {"Function": {"Tags": {"Env": {"Ref": "Environment"}, "Project": {"Ref": "ProjectName"}}}}, + "Resources": {"MySNSTopic": {"Type": "AWS::SNS::Topic"}}, + } + + model = parse(template).resolve() + + assert model.Globals.get("Function").get("Tags").get("Env") == "development" + assert model.Globals.get("Function").get("Tags").get("Project") == "my-project"