From 05f404709edb8c0e6e84f2b2c0ba732d8cb08592 Mon Sep 17 00:00:00 2001 From: Momo Kornher Date: Wed, 25 Oct 2023 15:55:05 +0100 Subject: [PATCH] feat(service-spec-importers): registry schema importers accept patches (#640) We want to move the service patches from the generic importers package to the specific model package. Generic patches stay with the importers. To support this, the two registry schema importers now accept an optional patcher. With that done, we can move the (now unused) service patches to aws-service-spec and use them in the implementation over there. --------- Signed-off-by: github-actions Co-authored-by: github-actions --- .projenrc.ts | 6 +- packages/@aws-cdk/aws-service-spec/.npmignore | 2 +- .../aws-service-spec/.projen/tasks.json | 2 +- .../{scripts => build}/augmentations.ts | 0 .../{scripts => build}/build-db.ts | 0 .../{scripts => build}/full-database.ts | 6 +- .../build/patches/registry-patches.ts | 10 +++ .../build}/patches/sam-patches.ts | 38 ++++----- .../patches/service-patches/autoscaling.ts | 4 +- .../build}/patches/service-patches/batch.ts | 4 +- .../patches/service-patches/cloudformation.ts | 4 +- .../patches/service-patches/codebuild.ts | 4 +- .../build}/patches/service-patches/cognito.ts | 6 +- .../build}/patches/service-patches/config.ts | 4 +- .../build}/patches/service-patches/core.ts | 82 +++++++++++-------- .../build}/patches/service-patches/dms.ts | 7 +- .../patches/service-patches/elasticsearch.ts | 4 +- .../build}/patches/service-patches/index.ts | 0 .../patches/service-patches/iot1click.ts | 4 +- .../patches/service-patches/opensearch.ts | 4 +- .../build}/patches/service-patches/rds.ts | 4 +- .../build}/patches/service-patches/s3.ts | 4 +- .../patches/service-patches/sagemaker.ts | 4 +- .../build}/patches/service-patches/wafv2.ts | 4 +- .../{scripts => build}/scrutinies.ts | 0 .../aws-service-spec/tsconfig.dev.json | 2 +- .../service-spec-importers/src/db-builder.ts | 14 +++- .../service-spec-importers/src/index.ts | 3 + .../loaders/load-cloudformation-registry.ts | 6 +- .../src/loaders/load-sam-schema.ts | 11 ++- .../src/patches/index.ts | 2 + .../src/patches/registry-patches.ts | 35 ++++---- 32 files changed, 158 insertions(+), 122 deletions(-) rename packages/@aws-cdk/aws-service-spec/{scripts => build}/augmentations.ts (100%) rename packages/@aws-cdk/aws-service-spec/{scripts => build}/build-db.ts (100%) rename packages/@aws-cdk/aws-service-spec/{scripts => build}/full-database.ts (87%) create mode 100644 packages/@aws-cdk/aws-service-spec/build/patches/registry-patches.ts rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/sam-patches.ts (64%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/autoscaling.ts (68%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/batch.ts (79%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/cloudformation.ts (78%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/codebuild.ts (90%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/cognito.ts (90%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/config.ts (88%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/core.ts (72%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/dms.ts (61%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/elasticsearch.ts (78%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/index.ts (100%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/iot1click.ts (90%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/opensearch.ts (78%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/rds.ts (52%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/s3.ts (65%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/sagemaker.ts (88%) rename packages/@aws-cdk/{service-spec-importers/src => aws-service-spec/build}/patches/service-patches/wafv2.ts (86%) rename packages/@aws-cdk/aws-service-spec/{scripts => build}/scrutinies.ts (100%) create mode 100644 packages/@aws-cdk/service-spec-importers/src/patches/index.ts diff --git a/.projenrc.ts b/.projenrc.ts index 6a4d10012..ebc6bafbb 100644 --- a/.projenrc.ts +++ b/.projenrc.ts @@ -156,12 +156,12 @@ const awsServiceSpec = new TypeScriptWorkspace({ releasableCommits: pj.ReleasableCommits.featuresAndFixes('. ../service-spec-types ../../../sources'), }); -awsServiceSpec.tsconfigDev.addInclude('scripts'); +awsServiceSpec.tsconfigDev.addInclude('build'); // Needs to be added to 'compile' task, because the integ tests will 'compile' everything (but not run the tests and linter). awsServiceSpec.compileTask.prependSpawn( awsServiceSpec.tasks.addTask('build:db', { - exec: `ts-node scripts/build-db.ts`, + exec: `ts-node build/build-db.ts`, }), ); @@ -169,7 +169,7 @@ awsServiceSpec.gitignore.addPatterns('db.json'); awsServiceSpec.gitignore.addPatterns('db.json.gz'); awsServiceSpec.gitignore.addPatterns('build-report'); awsServiceSpec.npmignore?.addPatterns('build-report'); -awsServiceSpec.npmignore?.addPatterns('/scripts/'); +awsServiceSpec.npmignore?.addPatterns('/build/'); // Add integration test with aws-cdk new AwsCdkIntegrationTest(repo, { diff --git a/packages/@aws-cdk/aws-service-spec/.npmignore b/packages/@aws-cdk/aws-service-spec/.npmignore index ca609fe4e..71f952e35 100644 --- a/packages/@aws-cdk/aws-service-spec/.npmignore +++ b/packages/@aws-cdk/aws-service-spec/.npmignore @@ -24,4 +24,4 @@ tsconfig.tsbuildinfo /.prettierignore /.prettierrc.json build-report -/scripts/ +/build/ diff --git a/packages/@aws-cdk/aws-service-spec/.projen/tasks.json b/packages/@aws-cdk/aws-service-spec/.projen/tasks.json index e77aea1de..97ab20339 100644 --- a/packages/@aws-cdk/aws-service-spec/.projen/tasks.json +++ b/packages/@aws-cdk/aws-service-spec/.projen/tasks.json @@ -25,7 +25,7 @@ "name": "build:db", "steps": [ { - "exec": "ts-node scripts/build-db.ts" + "exec": "ts-node build/build-db.ts" } ] }, diff --git a/packages/@aws-cdk/aws-service-spec/scripts/augmentations.ts b/packages/@aws-cdk/aws-service-spec/build/augmentations.ts similarity index 100% rename from packages/@aws-cdk/aws-service-spec/scripts/augmentations.ts rename to packages/@aws-cdk/aws-service-spec/build/augmentations.ts diff --git a/packages/@aws-cdk/aws-service-spec/scripts/build-db.ts b/packages/@aws-cdk/aws-service-spec/build/build-db.ts similarity index 100% rename from packages/@aws-cdk/aws-service-spec/scripts/build-db.ts rename to packages/@aws-cdk/aws-service-spec/build/build-db.ts diff --git a/packages/@aws-cdk/aws-service-spec/scripts/full-database.ts b/packages/@aws-cdk/aws-service-spec/build/full-database.ts similarity index 87% rename from packages/@aws-cdk/aws-service-spec/scripts/full-database.ts rename to packages/@aws-cdk/aws-service-spec/build/full-database.ts index 28effa1f7..0d5f2d7a2 100644 --- a/packages/@aws-cdk/aws-service-spec/scripts/full-database.ts +++ b/packages/@aws-cdk/aws-service-spec/build/full-database.ts @@ -3,6 +3,8 @@ import { SpecDatabase } from '@aws-cdk/service-spec-types'; import { DatabaseBuilder, DatabaseBuilderOptions, ReportAudience } from '@aws-cdk/service-spec-importers'; import { Augmentations } from './augmentations'; import { Scrutinies } from './scrutinies'; +import { patchSamTemplateSpec } from './patches/sam-patches'; +import { patchCloudFormationRegistry } from './patches/registry-patches'; const SOURCES = path.join(__dirname, '../../../../sources'); @@ -14,8 +16,8 @@ export class FullDatabase extends DatabaseBuilder { this.importCloudFormationResourceSpec(path.join(SOURCES, 'CloudFormationResourceSpecification')) .importSamResourceSpec(path.join(SOURCES, 'CloudFormationResourceSpecification/us-east-1/100_sam')) - .importCloudFormationRegistryResources(path.join(SOURCES, 'CloudFormationSchema')) - .importSamJsonSchema(path.join(SOURCES, 'SAMSpec/sam.schema.json')) + .importCloudFormationRegistryResources(path.join(SOURCES, 'CloudFormationSchema'), patchCloudFormationRegistry) + .importSamJsonSchema(path.join(SOURCES, 'SAMSpec/sam.schema.json'), patchSamTemplateSpec) .importCloudFormationDocs(path.join(SOURCES, 'CloudFormationDocumentation/CloudFormationDocumentation.json')) .importStatefulResources(path.join(SOURCES, 'StatefulResources/StatefulResources.json')) .importCannedMetrics( diff --git a/packages/@aws-cdk/aws-service-spec/build/patches/registry-patches.ts b/packages/@aws-cdk/aws-service-spec/build/patches/registry-patches.ts new file mode 100644 index 000000000..eb0f8c2d0 --- /dev/null +++ b/packages/@aws-cdk/aws-service-spec/build/patches/registry-patches.ts @@ -0,0 +1,10 @@ +import { EXCEPTIONS_PATCHERS } from './service-patches'; +import { patching, patches } from '@aws-cdk/service-spec-importers'; + +/** + * Patchers that apply to the CloudFormation Registry source files + */ +export const patchCloudFormationRegistry = patching.makeCompositePatcher( + patches.patchCloudFormationRegistry, + ...EXCEPTIONS_PATCHERS, +); diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/sam-patches.ts b/packages/@aws-cdk/aws-service-spec/build/patches/sam-patches.ts similarity index 64% rename from packages/@aws-cdk/service-spec-importers/src/patches/sam-patches.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/sam-patches.ts index efcf9ade2..1789ba18f 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/sam-patches.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/sam-patches.ts @@ -1,34 +1,34 @@ -import { normalizeJsonSchema } from './json-schema-patches'; +import { patching, patches, types } from '@aws-cdk/service-spec-importers'; import { addDefinitions, replaceDefinition, replaceDefinitionProperty } from './service-patches/core'; -import { JsonObjectLens, JsonObjectPatcher, Patcher, Reason, makeCompositePatcher, onlyObjects } from '../patching'; -import { jsonschema } from '../types'; -const serverlessApi: Patcher = (lens) => { +const serverlessApi: patching.Patcher = (lens) => { replaceSamResourceProperty( 'AWS::Serverless::Api', 'EndpointConfiguration', { anyOf: [{ $ref: '#/definitions/AWS::Serverless::Api.EndpointConfiguration' }, { type: 'string' }], }, - Reason.backwardsCompat('Make the EndpointConfiguration property of AWS::Serverless::Api have a union type'), + patching.Reason.backwardsCompat( + 'Make the EndpointConfiguration property of AWS::Serverless::Api have a union type', + ), )(lens); replaceSamResourceProperty( 'AWS::Serverless::Api', 'GatewayResponses', { type: 'object' }, - Reason.backwardsCompat('Make the GatewayResponses property of AWS::Serverless::Api accept JSON'), + patching.Reason.backwardsCompat('Make the GatewayResponses property of AWS::Serverless::Api accept JSON'), )(lens); replaceSamResourceProperty( 'AWS::Serverless::Api', 'Models', { type: 'object' }, - Reason.backwardsCompat('Make the Models property of AWS::Serverless::Api accept JSON'), + patching.Reason.backwardsCompat('Make the Models property of AWS::Serverless::Api accept JSON'), )(lens); }; -const serverlessFunction: Patcher = (lens) => { - const hooksReason = Reason.sourceIssue('Use of pattern properties but type is actually well-known.'); +const serverlessFunction: patching.Patcher = (lens) => { + const hooksReason = patching.Reason.sourceIssue('Use of pattern properties but type is actually well-known.'); replaceDefinitionProperty( 'AWS::Serverless::Function.DeploymentPreference', @@ -59,7 +59,7 @@ const serverlessFunction: Patcher = (lens) => { { type: 'object', }, - Reason.backwardsCompat( + patching.Reason.backwardsCompat( 'This was once typed as Json, and adding types now is a breaking change. Keep them as Json forever', ), )(lens); @@ -74,18 +74,18 @@ const serverlessFunction: Patcher = (lens) => { type: 'object', required: ['SkillId'], }, - Reason.sourceIssue('SAM docs claim this is optional, but it is the only possible property'), + patching.Reason.sourceIssue('SAM docs claim this is optional, but it is the only possible property'), )(lens); }; -const serverlessStateMachine: Patcher = (lens) => { +const serverlessStateMachine: patching.Patcher = (lens) => { replaceDefinitionProperty( 'AWS::Serverless::StateMachine.IAMPolicyDocument', 'Statement', { type: 'object', }, - Reason.backwardsCompat( + patching.Reason.backwardsCompat( 'This was once typed as Json, and adding types now is a breaking change. Keep them as Json forever', ), )(lens); @@ -94,9 +94,9 @@ const serverlessStateMachine: Patcher = (lens) => { /** * Patchers that apply to the SAM Template spec file */ -export const patchSamTemplateSpec = makeCompositePatcher( - normalizeJsonSchema, - onlyObjects(makeCompositePatcher(serverlessApi, serverlessFunction, serverlessStateMachine)), +export const patchSamTemplateSpec = patching.makeCompositePatcher( + patches.normalizeJsonSchema, + patching.onlyObjects(patching.makeCompositePatcher(serverlessApi, serverlessFunction, serverlessStateMachine)), ); /** @@ -107,9 +107,9 @@ export const patchSamTemplateSpec = makeCompositePatcher( function replaceSamResourceProperty( resource: string, propertyName: string, - newSchema: jsonschema.Schema, - reason: Reason, -): JsonObjectPatcher { + newSchema: types.jsonschema.Schema, + reason: patching.Reason, +): patching.JsonObjectPatcher { return (lens) => { if (lens.jsonPointer === `/definitions/${resource}/properties/Properties/properties/${propertyName}`) { lens.replaceValue(reason.reason, newSchema); diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/autoscaling.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/autoscaling.ts similarity index 68% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/autoscaling.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/autoscaling.ts index 56e98120b..688d3513f 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/autoscaling.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/autoscaling.ts @@ -1,7 +1,7 @@ import { forResource, fp, registerServicePatches, removeResourceProperty } from './core'; -import { Reason } from '../../patching'; +import { patching } from '@aws-cdk/service-spec-importers'; -const reason = Reason.sourceIssue('Remove (presumed wrongly included) autoscaling group attribute'); +const reason = patching.Reason.sourceIssue('Remove (presumed wrongly included) autoscaling group attribute'); registerServicePatches( fp.removeFromReadOnlyProperties('AWS::AutoScaling::AutoScalingGroup', ['LaunchTemplateSpecification'], reason), diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/batch.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/batch.ts similarity index 79% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/batch.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/batch.ts index 12972a895..0d3326fad 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/batch.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/batch.ts @@ -1,9 +1,9 @@ import { forResource, registerServicePatches, renameDefinition } from './core'; -import { Reason } from '../../patching'; +import { patching } from '@aws-cdk/service-spec-importers'; registerServicePatches( forResource('AWS::Batch::JobDefinition', (lens) => { - const reason = Reason.upstreamTypeNameChange(); + const reason = patching.Reason.upstreamTypeNameChange(); renameDefinition('EksEmptyDir', 'EmptyDir', reason)(lens); renameDefinition('EksHostPath', 'HostPath', reason)(lens); diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/cloudformation.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/cloudformation.ts similarity index 78% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/cloudformation.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/cloudformation.ts index 5a83a29b7..8d7e5817d 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/cloudformation.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/cloudformation.ts @@ -1,7 +1,7 @@ import { forResource, registerServicePatches, removeResourceProperty } from './core'; -import { Reason } from '../../patching'; +import { patching } from '@aws-cdk/service-spec-importers'; -const reason = Reason.sourceIssue('Property is only supported by the CCAPI'); +const reason = patching.Reason.sourceIssue('Property is only supported by the CCAPI'); registerServicePatches( forResource('AWS::CloudFormation::Stack', (lens) => { diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/codebuild.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/codebuild.ts similarity index 90% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/codebuild.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/codebuild.ts index 44ac3167b..53128a8b4 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/codebuild.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/codebuild.ts @@ -1,12 +1,12 @@ import { addDefinitions, forResource, registerServicePatches, replaceDefinition } from './core'; -import { Reason } from '../../patching'; +import { patching } from '@aws-cdk/service-spec-importers'; /** * Add missing types for AWS::CodeBuild::Project */ registerServicePatches( forResource('AWS::CodeBuild::Project', (lens) => { - const reason = Reason.sourceIssue( + const reason = patching.Reason.sourceIssue( 'The elements of AWS::CodeBuild::Project.Triggers.FilterGroups used to be well-typed in the Resource Specification. In the Resource Schema it is incorrectly an untyped object.', ); diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/cognito.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/cognito.ts similarity index 90% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/cognito.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/cognito.ts index a5d5217af..5e3ec56bb 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/cognito.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/cognito.ts @@ -1,5 +1,5 @@ import { addDefinitions, forResource, registerServicePatches, replaceResourceProperty } from './core'; -import { Reason } from '../../patching'; +import { patching } from '@aws-cdk/service-spec-importers'; /** * Make the use of the AWS::Cognito::IdentityPoolRoleAttachment.RoleMapings property safer @@ -19,7 +19,7 @@ registerServicePatches( type: 'object', additionalProperties: { $ref: '#/definitions/RoleMapping' }, }, - Reason.other('Make the use of RoleMappings more type safe'), + patching.Reason.other('Make the use of RoleMappings more type safe'), )(lens); addDefinitions( @@ -67,7 +67,7 @@ registerServicePatches( }, }, - Reason.other('Make the use of RoleMappings more type safe'), + patching.Reason.other('Make the use of RoleMappings more type safe'), )(lens); }), ); diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/config.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/config.ts similarity index 88% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/config.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/config.ts index 6567346b6..0c49f849c 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/config.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/config.ts @@ -1,9 +1,9 @@ import { addDefinitions, forResource, registerServicePatches, replaceResourceProperty } from './core'; -import { Reason } from '../../patching'; +import { patching } from '@aws-cdk/service-spec-importers'; registerServicePatches( forResource('AWS::Config::RemediationConfiguration', (lens) => { - const reason = Reason.sourceIssue('Unused property type in Spec, now missing in Schema'); + const reason = patching.Reason.sourceIssue('Unused property type in Spec, now missing in Schema'); replaceResourceProperty( 'Parameters', { diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/core.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/core.ts similarity index 72% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/core.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/core.ts index 25ee4f509..8dc7de447 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/core.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/core.ts @@ -1,12 +1,11 @@ -import { JsonLens, JsonLensPatcher, JsonObjectPatcher, Patcher, Reason, fp as fun, isRoot } from '../../patching'; -import { CloudFormationRegistryResource, jsonschema } from '../../types'; +import { patching, types } from '@aws-cdk/service-spec-importers'; -export const SERVICE_PATCHERS: Array = []; +export const SERVICE_PATCHERS: Array = []; /** * Register an unnamed exception patcher */ -export function registerServicePatches(...patcher: JsonLensPatcher[]) { +export function registerServicePatches(...patcher: patching.JsonLensPatcher[]) { SERVICE_PATCHERS.push(...patcher); } @@ -15,13 +14,13 @@ export function registerServicePatches(...patcher: JsonLensPatcher[]) { * * It will still be invoked at every JSON node in that document. */ -export function forResource(resource: string, patcher: JsonObjectPatcher): JsonLensPatcher { +export function forResource(resource: string, patcher: patching.JsonObjectPatcher): patching.JsonLensPatcher { return (lens) => { const root = lens.rootPath[0]; if ( lens.isJsonObject() && root.isJsonObject() && - (root.value as unknown as CloudFormationRegistryResource).typeName === resource + (root.value as unknown as types.CloudFormationRegistryResource).typeName === resource ) { patcher(lens); } @@ -35,9 +34,9 @@ export function forResource(resource: string, patcher: JsonObjectPatcher): JsonL */ export function replaceResourceProperty( propertyName: string, - newSchema: jsonschema.Schema, - reason: Reason, -): JsonObjectPatcher { + newSchema: types.jsonschema.Schema, + reason: patching.Reason, +): patching.JsonObjectPatcher { return (lens) => { if (lens.jsonPointer === `/properties/${propertyName}`) { lens.replaceValue(reason.reason, newSchema); @@ -50,7 +49,7 @@ export function replaceResourceProperty( * * NOTE: returns a new patcher. Still needs to be applied to a lens. */ -export function removeResourceProperty(propertyName: string, reason: Reason): JsonObjectPatcher { +export function removeResourceProperty(propertyName: string, reason: patching.Reason): patching.JsonObjectPatcher { return (lens) => { if (lens.jsonPointer === `/properties`) { lens.removeProperty(reason.reason, propertyName); @@ -66,9 +65,9 @@ export function removeResourceProperty(propertyName: string, reason: Reason): Js export function replaceDefinitionProperty( definitionName: string, propertyName: string, - newSchema: jsonschema.Schema, - reason: Reason, -): JsonObjectPatcher { + newSchema: types.jsonschema.Schema, + reason: patching.Reason, +): patching.JsonObjectPatcher { return (lens) => { if (lens.jsonPointer === `/definitions/${definitionName}/properties/${propertyName}`) { lens.replaceValue(reason.reason, newSchema); @@ -81,7 +80,11 @@ export function replaceDefinitionProperty( * * NOTE: returns a new patcher. Still needs to be applied to a lens. */ -export function renameDefinition(oldName: string, newName: string, reason: Reason): JsonObjectPatcher { +export function renameDefinition( + oldName: string, + newName: string, + reason: patching.Reason, +): patching.JsonObjectPatcher { return (lens) => { if (lens.jsonPointer === `/definitions`) { lens.renameProperty(reason.reason, oldName, newName); @@ -104,7 +107,11 @@ export function renameDefinition(oldName: string, newName: string, reason: Reaso * * NOTE: returns a new patcher. Still needs to be applied to a lens. */ -export function replaceDefinition(definition: string, schema: jsonschema.Schema, reason: Reason): JsonObjectPatcher { +export function replaceDefinition( + definition: string, + schema: types.jsonschema.Schema, + reason: patching.Reason, +): patching.JsonObjectPatcher { return (lens) => { if (lens.jsonPointer === `/definitions/${definition}`) { lens.replaceValue(reason.reason, schema); @@ -117,9 +124,12 @@ export function replaceDefinition(definition: string, schema: jsonschema.Schema, * * NOTE: returns a new patcher. Still needs to be applied to a lens. */ -export function addDefinitions(definitions: Record, reason: Reason): JsonObjectPatcher { +export function addDefinitions( + definitions: Record, + reason: patching.Reason, +): patching.JsonObjectPatcher { return (lens) => { - if (isRoot(lens) && lens.value.definitions === undefined) { + if (patching.isRoot(lens) && lens.value.definitions === undefined) { // No '/definitions' in this type lens.addProperty(reason.reason, 'definitions', definitions); } else if (lens.jsonPointer === '/definitions') { @@ -143,13 +153,13 @@ export namespace fp { export function patchResourceAt( resource: TypeName, pointer: string, - reason: Reason, - patch: fun.Patch, - ): Patcher { + reason: patching.Reason, + patch: patching.fp.Patch, + ): patching.Patcher { return (lens) => { const root = lens.rootPath[0]; - if ((root.value as unknown as CloudFormationRegistryResource).typeName === resource) { - fun.patchAt(pointer, reason, patch)(lens); + if ((root.value as unknown as types.CloudFormationRegistryResource).typeName === resource) { + patching.fp.patchAt(pointer, reason, patch)(lens); } }; } @@ -158,11 +168,11 @@ export namespace fp { * Patch a resource at the root */ export function patchResource< - Tree extends CloudFormationRegistryResource & { + Tree extends types.CloudFormationRegistryResource & { typeName: TypeName; }, TypeName extends string = string, - >(resource: TypeName, reason: Reason, patch: fun.Patch): Patcher { + >(resource: TypeName, reason: patching.Reason, patch: patching.fp.Patch): patching.Patcher { return patchResourceAt(resource, '', reason, patch); } @@ -173,9 +183,9 @@ export namespace fp { export function removeFromReadOnlyProperties( resource: TypeName, remove: string[], - reason: Reason, - ): Patcher { - return patchResourceAt( + reason: patching.Reason, + ): patching.Patcher { + return patchResourceAt( resource, '/readOnlyProperties', reason, @@ -198,9 +208,9 @@ export namespace fp { export function addReadOnlyProperties( resource: TypeName, additional: string[], - reason: Reason, - ): Patcher { - return patchResourceAt( + reason: patching.Reason, + ): patching.Patcher { + return patchResourceAt( resource, '/readOnlyProperties', reason, @@ -224,9 +234,9 @@ export namespace fp { replace: { [oldName: string]: string; }, - reason: Reason, - ): Patcher { - return patchResourceAt( + reason: patching.Reason, + ): patching.Patcher { + return patchResourceAt( resource, '/readOnlyProperties', reason, @@ -250,12 +260,12 @@ export namespace fp { rename: { [oldName: string]: string; }, - reason: Reason, - ): Patcher { + reason: patching.Reason, + ): patching.Patcher { return (lens) => { const root = lens.rootPath[0]; if ( - (root.value as unknown as CloudFormationRegistryResource).typeName === resource && + (root.value as unknown as types.CloudFormationRegistryResource).typeName === resource && lens.jsonPointer === '/properties' && lens.isJsonObject() ) { diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/dms.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/dms.ts similarity index 61% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/dms.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/dms.ts index a074ee5df..61c0170d9 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/dms.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/dms.ts @@ -1,12 +1,11 @@ import { fp, registerServicePatches } from './core'; -import { Reason } from '../../patching'; -import { CloudFormationRegistryResource } from '../../types'; +import { patching, types } from '@aws-cdk/service-spec-importers'; registerServicePatches( - fp.patchResourceAt( + fp.patchResourceAt( 'AWS::DMS::ReplicationConfig', '/readOnlyProperties', - Reason.sourceIssue('Incorrect case. Got upper case `/Properties` instead of `/properties'), + patching.Reason.sourceIssue('Incorrect case. Got upper case `/Properties` instead of `/properties'), (readOnlyProperties = []) => { for (const [idx, prop] of readOnlyProperties.entries()) { if (prop.startsWith('/Properties')) { diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/elasticsearch.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/elasticsearch.ts similarity index 78% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/elasticsearch.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/elasticsearch.ts index ac692aed2..63b1c8e4c 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/elasticsearch.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/elasticsearch.ts @@ -1,11 +1,11 @@ import { fp, registerServicePatches } from './core'; -import { Reason } from '../../patching'; +import { patching } from '@aws-cdk/service-spec-importers'; registerServicePatches( fp.removeFromReadOnlyProperties( 'AWS::Elasticsearch::Domain', ['DomainArn'], - Reason.other( + patching.Reason.other( 'Remove the deprecated attribute DomainArn, as the new preferred attribute Arn maps to the same name in the generated code', ), ), diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/index.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/index.ts similarity index 100% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/index.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/index.ts diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/iot1click.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/iot1click.ts similarity index 90% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/iot1click.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/iot1click.ts index 63a888fd8..c68217591 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/iot1click.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/iot1click.ts @@ -1,12 +1,12 @@ import { addDefinitions, forResource, registerServicePatches, replaceDefinitionProperty } from './core'; -import { Reason } from '../../patching'; +import { patching } from '@aws-cdk/service-spec-importers'; /** * We enhance the types for IoT project */ registerServicePatches( forResource('AWS::IoT1Click::Project', (lens) => { - const reason = Reason.other( + const reason = patching.Reason.other( 'Set type of AWS::IoT1Click::Project.PlacementTemplate.DeviceTemplates to Map', ); diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/opensearch.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/opensearch.ts similarity index 78% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/opensearch.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/opensearch.ts index ba86239c6..23192dfe8 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/opensearch.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/opensearch.ts @@ -1,11 +1,11 @@ import { fp, registerServicePatches } from './core'; -import { Reason } from '../../patching'; +import { patching } from '@aws-cdk/service-spec-importers'; registerServicePatches( fp.removeFromReadOnlyProperties( 'AWS::OpenSearchService::Domain', ['DomainArn'], - Reason.other( + patching.Reason.other( 'Remove the DomainArn attribute of AWS::OpenSearchService::Domain resources, as it is unsupported by CloudFormation', ), ), diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/rds.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/rds.ts similarity index 52% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/rds.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/rds.ts index 215a84a60..31e896473 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/rds.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/rds.ts @@ -1,10 +1,10 @@ import { fp, registerServicePatches } from './core'; -import { Reason } from '../../patching'; +import { patching } from '@aws-cdk/service-spec-importers'; registerServicePatches( fp.addReadOnlyProperties( 'AWS::RDS::DBCluster', ['ReadEndpoint'], - Reason.sourceIssue('ReadEndpoint should be listed in readOnlyProperties.'), + patching.Reason.sourceIssue('ReadEndpoint should be listed in readOnlyProperties.'), ), ); diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/s3.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/s3.ts similarity index 65% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/s3.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/s3.ts index b61aa97c4..e4a83e84c 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/s3.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/s3.ts @@ -1,7 +1,7 @@ import { forResource, registerServicePatches, replaceDefinitionProperty } from './core'; -import { Reason } from '../../patching'; +import { patching } from '@aws-cdk/service-spec-importers'; -const reason = Reason.sourceIssue('Integer property incorrectly defined as string that only allows number characters'); +const reason = patching.Reason.sourceIssue('Integer property incorrectly defined as string that only allows number characters'); registerServicePatches( forResource('AWS::S3::Bucket', (lens) => { diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/sagemaker.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/sagemaker.ts similarity index 88% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/sagemaker.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/sagemaker.ts index 6677996ad..55380bbaa 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/sagemaker.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/sagemaker.ts @@ -1,9 +1,9 @@ import { addDefinitions, forResource, registerServicePatches, replaceDefinitionProperty } from './core'; -import { Reason } from '../../patching'; +import { patching } from '@aws-cdk/service-spec-importers'; registerServicePatches( forResource('AWS::SageMaker::ModelCard', (lens) => { - const reason = Reason.upstreamTypeNameChange('Was a single type, is now multiple XOR types.'); + const reason = patching.Reason.upstreamTypeNameChange('Was a single type, is now multiple XOR types.'); replaceDefinitionProperty( 'MetricGroup', diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/wafv2.ts b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/wafv2.ts similarity index 86% rename from packages/@aws-cdk/service-spec-importers/src/patches/service-patches/wafv2.ts rename to packages/@aws-cdk/aws-service-spec/build/patches/service-patches/wafv2.ts index 483e1c5af..00ca26508 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/service-patches/wafv2.ts +++ b/packages/@aws-cdk/aws-service-spec/build/patches/service-patches/wafv2.ts @@ -1,9 +1,9 @@ import { forResource, registerServicePatches, renameDefinition } from './core'; -import { Reason } from '../../patching'; +import { patching } from '@aws-cdk/service-spec-importers'; registerServicePatches( forResource('AWS::WAFv2::RuleGroup', (lens) => { - const reason = Reason.other( + const reason = patching.Reason.other( 'Reverting property type names from FooAction to Foo, which were introduced as part of this PR: https://github.com/aws/aws-cdk/pull/23984', ); diff --git a/packages/@aws-cdk/aws-service-spec/scripts/scrutinies.ts b/packages/@aws-cdk/aws-service-spec/build/scrutinies.ts similarity index 100% rename from packages/@aws-cdk/aws-service-spec/scripts/scrutinies.ts rename to packages/@aws-cdk/aws-service-spec/build/scrutinies.ts diff --git a/packages/@aws-cdk/aws-service-spec/tsconfig.dev.json b/packages/@aws-cdk/aws-service-spec/tsconfig.dev.json index 4e7806c03..0f39d6050 100644 --- a/packages/@aws-cdk/aws-service-spec/tsconfig.dev.json +++ b/packages/@aws-cdk/aws-service-spec/tsconfig.dev.json @@ -31,7 +31,7 @@ ".projenrc.js", "src/**/*.ts", "test/**/*.ts", - "scripts" + "build" ], "exclude": [ "node_modules" diff --git a/packages/@aws-cdk/service-spec-importers/src/db-builder.ts b/packages/@aws-cdk/service-spec-importers/src/db-builder.ts index a6f1d4d3b..ec81327c7 100644 --- a/packages/@aws-cdk/service-spec-importers/src/db-builder.ts +++ b/packages/@aws-cdk/service-spec-importers/src/db-builder.ts @@ -16,6 +16,7 @@ import { loadSamSchema, loadSamSpec, } from './loaders'; +import { JsonLensPatcher } from './patching'; import { ProblemReport, ReportAudience } from './report'; export interface DatabaseBuilderOptions { @@ -107,12 +108,13 @@ export class DatabaseBuilder { /** * Import the (modern) registry spec from CloudFormation */ - public importCloudFormationRegistryResources(schemaDirectory: string) { + public importCloudFormationRegistryResources(schemaDirectory: string, patcher?: JsonLensPatcher) { return this.addSourceImporter(async (db, report) => { const regions = await loadDefaultCloudFormationRegistryResources(schemaDirectory, { ...this.options, report, failureAudience: this.defaultProblemGrouping, + patcher, }); for (const region of regions) { for (const resource of region.resources) { @@ -130,9 +132,15 @@ export class DatabaseBuilder { /** * Import the (modern) JSON schema spec from SAM */ - public importSamJsonSchema(filePath: string) { + public importSamJsonSchema(filePath: string, patcher?: JsonLensPatcher) { return this.addSourceImporter(async (db, report) => { - const samSchema = this.loadResult(await loadSamSchema(filePath, this.options), report); + const samSchema = this.loadResult( + await loadSamSchema(filePath, { + ...this.options, + patcher, + }), + report, + ); new SamResources({ db, samSchema, report }).import(); }); } diff --git a/packages/@aws-cdk/service-spec-importers/src/index.ts b/packages/@aws-cdk/service-spec-importers/src/index.ts index fa76d824d..61d9bccb5 100644 --- a/packages/@aws-cdk/service-spec-importers/src/index.ts +++ b/packages/@aws-cdk/service-spec-importers/src/index.ts @@ -1,3 +1,6 @@ export * from './db-builder'; export * from './db-diff'; export * from './report/problem-report'; +export * as patching from './patching'; +export * as patches from './patches'; +export * as types from './types'; diff --git a/packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudformation-registry.ts b/packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudformation-registry.ts index d483fb7e3..4eb978210 100644 --- a/packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudformation-registry.ts +++ b/packages/@aws-cdk/service-spec-importers/src/loaders/load-cloudformation-registry.ts @@ -3,7 +3,8 @@ import * as util from 'util'; import { isSuccess, Result } from '@cdklabs/tskb'; import * as _glob from 'glob'; import { Loader, LoadResult, LoadSourceOptions } from './loader'; -import { patchCloudFormationRegistry } from '../patches/registry-patches'; +import { patchCloudFormationRegistry } from '../patches'; +import { JsonLensPatcher } from '../patching'; import { ProblemReport, ReportAudience } from '../report'; import { CloudFormationRegistryResource } from '../types'; @@ -12,6 +13,7 @@ const glob = util.promisify(_glob.glob); interface LoadCloudFormationRegistrySourceOptions extends LoadSourceOptions { readonly report: ProblemReport; readonly failureAudience: ReportAudience; + readonly patcher?: JsonLensPatcher; } export function loadCloudFormationRegistryDirectory( @@ -22,7 +24,7 @@ export function loadCloudFormationRegistryDirectory( 'CloudFormationRegistryResource.schema.json', { mustValidate: options.validate, - patcher: patchCloudFormationRegistry, + patcher: options.patcher ?? patchCloudFormationRegistry, errorRootDirectory: baseDir, }, ); diff --git a/packages/@aws-cdk/service-spec-importers/src/loaders/load-sam-schema.ts b/packages/@aws-cdk/service-spec-importers/src/loaders/load-sam-schema.ts index cc06722c9..b1f7e2437 100644 --- a/packages/@aws-cdk/service-spec-importers/src/loaders/load-sam-schema.ts +++ b/packages/@aws-cdk/service-spec-importers/src/loaders/load-sam-schema.ts @@ -1,18 +1,23 @@ import { assertSuccess } from '@cdklabs/tskb'; import { Loader, LoadResult, LoadSourceOptions } from './loader'; -import { patchSamTemplateSpec } from '../patches/sam-patches'; +import { normalizeJsonSchema } from '../patches'; +import { JsonLensPatcher } from '../patching'; import { SamTemplateSchema } from '../types'; +interface LoadSamSchemaSourceOptions extends LoadSourceOptions { + readonly patcher?: JsonLensPatcher; +} + /** * Load the new SAM (json) schema */ export async function loadSamSchema( filePath: string, - options: LoadSourceOptions = {}, + options: LoadSamSchemaSourceOptions = {}, ): Promise> { const loader = await Loader.fromSchemaFile('SamTemplateSchema.schema.json', { mustValidate: options.validate, - patcher: patchSamTemplateSpec, + patcher: options.patcher ?? normalizeJsonSchema, }); const result = await loader.loadFile(filePath); diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/index.ts b/packages/@aws-cdk/service-spec-importers/src/patches/index.ts new file mode 100644 index 000000000..3a461a5c1 --- /dev/null +++ b/packages/@aws-cdk/service-spec-importers/src/patches/index.ts @@ -0,0 +1,2 @@ +export * from './json-schema-patches'; +export { patchCloudFormationRegistry } from './registry-patches'; diff --git a/packages/@aws-cdk/service-spec-importers/src/patches/registry-patches.ts b/packages/@aws-cdk/service-spec-importers/src/patches/registry-patches.ts index 2ab63f537..75bf025b9 100644 --- a/packages/@aws-cdk/service-spec-importers/src/patches/registry-patches.ts +++ b/packages/@aws-cdk/service-spec-importers/src/patches/registry-patches.ts @@ -5,7 +5,6 @@ */ import canonicalize from 'canonicalize'; import { normalizeJsonSchema } from './json-schema-patches'; -import { EXCEPTIONS_PATCHERS } from './service-patches'; import { TypeKeyWitness, STRING_KEY_WITNESS, @@ -23,26 +22,22 @@ import { /** * Patchers that apply to the CloudFormation Registry source files */ -export const patchCloudFormationRegistry = makeCompositePatcher( - onlyObjects( - makeCompositePatcher( - normalizeJsonSchema, - replaceArrayLengthProps, - removeBooleanPatterns, - canonicalizeDefaultOnBoolean, - patchMinLengthOnInteger, - canonicalizeRegexInFormat, - markAsNonTaggable, - incorrectTagPropertyFormat, - noIncorrectDefaultType, - removeSuspiciousPatterns, - dropRedundantTypeOperatorsInMetricStream, - minMaxItemsOnObject, - makeKeywordDropper(), - ), +export const patchCloudFormationRegistry = onlyObjects( + makeCompositePatcher( + normalizeJsonSchema, + replaceArrayLengthProps, + removeBooleanPatterns, + canonicalizeDefaultOnBoolean, + patchMinLengthOnInteger, + canonicalizeRegexInFormat, + markAsNonTaggable, + incorrectTagPropertyFormat, + noIncorrectDefaultType, + removeSuspiciousPatterns, + dropRedundantTypeOperatorsInMetricStream, + minMaxItemsOnObject, + makeKeywordDropper(), ), - // Service patches might have to change arrays - ...EXCEPTIONS_PATCHERS, ); /**