feat(aws-service-spec): custom build using service-spec-importers (#635)

Updates `aws-service-spec` to run a custom build that is using the
public API from `service-spec-importers`.

Also adds a new build step to post the diff between `base` and `head`
branch to the PR.

.projenrc.ts

@@ -1,5 +1,5 @@
 import * as pj from 'projen';
-import { AwsCdkIntegrationTest, TypeScriptWorkspace, YarnMonorepo } from './projenrc';
+import { AwsCdkIntegrationTest, DiffDb, TypeScriptWorkspace, YarnMonorepo } from './projenrc';
 import { RegionalSource, Role, SingleSource, SourceProcessing } from './projenrc/update-sources';
 
 const workflowRunsOn = [
@@ -155,18 +155,21 @@ const awsServiceSpec = new TypeScriptWorkspace({
   // Also include changes to types and sources
   releasableCommits: pj.ReleasableCommits.featuresAndFixes('. ../service-spec-types ../../../sources'),
 });
+
+awsServiceSpec.tsconfigDev.addInclude('scripts');
+
 // Needs to be added to 'compile' task, because the integ tests will 'compile' everything (but not run the tests and linter).
 awsServiceSpec.compileTask.prependSpawn(
-  awsServiceSpec.tasks.addTask('generate', {
-    exec: `import-db db.json.gz --force`,
-    receiveArgs: true,
+  awsServiceSpec.tasks.addTask('build:db', {
+    exec: `ts-node scripts/build-db.ts`,
   }),
 );
 
 awsServiceSpec.gitignore.addPatterns('db.json');
 awsServiceSpec.gitignore.addPatterns('db.json.gz');
 awsServiceSpec.gitignore.addPatterns('build-report');
 awsServiceSpec.npmignore?.addPatterns('build-report');
+awsServiceSpec.npmignore?.addPatterns('/scripts/');
 
 // Add integration test with aws-cdk
 new AwsCdkIntegrationTest(repo, {
@@ -175,6 +178,13 @@ new AwsCdkIntegrationTest(repo, {
   serviceSpecTypes,
 });
 
+// Post diff of database
+new DiffDb(repo, {
+  workflowRunsOn,
+  serviceSpec: awsServiceSpec,
+  serviceSpecImporters,
+});
+
 // Update sources
 new SingleSource(repo, {
   name: 'documentation',

packages/@aws-cdk/aws-service-spec/scripts/build-db.ts

@@ -0,0 +1,33 @@
+import * as zlib from 'node:zlib';
+import { emptyDatabase } from '@aws-cdk/service-spec-types';
+import { FullDatabase } from './full-database';
+import { writeFile } from 'node:fs/promises';
+
+async function main() {
+  const outputFile = 'db.json.gz';
+  const buildReport = 'build-report';
+
+  process.stdout.write('Importing default sources... ');
+  const { db, report } = await new FullDatabase(emptyDatabase(), {
+    // @TODO: Switch this to 'true' once validation passes (this will never happen)
+    validate: false,
+    debug: Boolean(process.env.DEBUG),
+  }).build();
+  const importResult = report.totalCount ? `WARN (${report.totalCount} problems encountered)` : 'OK';
+  process.stdout.write(`${importResult}\n`);
+
+  process.stdout.write(`Writing database to file ${outputFile}... `);
+  const data = JSON.stringify(db.save());
+  await writeFile(outputFile, zlib.gzipSync(data), { encoding: 'binary' });
+  process.stdout.write('OK\n');
+
+  process.stdout.write(`Creating build report in ${buildReport}... `);
+  await report.write(buildReport);
+  process.stdout.write('OK\n');
+}
+
+main().catch((error) => {
+  process.exitCode = 1;
+  console.error('FAIL\n\n');
+  console.error(error);
+});

packages/@aws-cdk/aws-service-spec/scripts/full-database.ts

@@ -0,0 +1,41 @@
+import * as path from 'node:path';
+import { SpecDatabase } from '@aws-cdk/service-spec-types';
+import { DatabaseBuilder, DatabaseBuilderOptions, ReportAudience } from '@aws-cdk/service-spec-importers';
+import { Augmentations } from './augmentations';
+import { Scrutinies } from './scrutinies';
+
+const SOURCES = path.join(__dirname, '../../../../sources');
+
+export class FullDatabase extends DatabaseBuilder {
+  public defaultProblemGrouping = new ReportAudience('CDK_Team');
+
+  constructor(db: SpecDatabase, options: DatabaseBuilderOptions) {
+    super(db, options);
+
+    this.importCloudFormationResourceSpec(path.join(SOURCES, 'CloudFormationResourceSpecification'))
+      .importSamResourceSpec(path.join(SOURCES, 'CloudFormationResourceSpecification/us-east-1/100_sam'))
+      .importCloudFormationRegistryResources(path.join(SOURCES, 'CloudFormationSchema'))
+      .importSamJsonSchema(path.join(SOURCES, 'SAMSpec/sam.schema.json'))
+      .importCloudFormationDocs(path.join(SOURCES, 'CloudFormationDocumentation/CloudFormationDocumentation.json'))
+      .importStatefulResources(path.join(SOURCES, 'StatefulResources/StatefulResources.json'))
+      .importCannedMetrics(
+        path.join(SOURCES, 'CloudWatchConsoleServiceDirectory/CloudWatchConsoleServiceDirectory.json'),
+      )
+      .importScrutinies()
+      .importAugmentations();
+  }
+
+  /**
+   * Import Augmentations
+   */
+  public importAugmentations() {
+    return this.addSourceImporter(async (db) => new Augmentations(db).import());
+  }
+
+  /**
+   * Import Scrutinies
+   */
+  public importScrutinies() {
+    return this.addSourceImporter(async (db) => new Scrutinies(db).import());
+  }
+}

packages/@aws-cdk/aws-service-spec/src/index.ts

@@ -1,6 +1,6 @@
-import { promises as fs } from 'fs';
-import path from 'path';
-import { gunzipSync } from 'zlib';
+import { promises as fs } from 'node:fs';
+import * as path from 'node:path';
+import { gunzipSync } from 'node:zlib';
 import { emptyDatabase, SpecDatabase } from '@aws-cdk/service-spec-types';
 
 const DB_COMPRESSED = 'db.json.gz';

packages/@aws-cdk/service-spec-importers/README.md

@@ -26,8 +26,6 @@ new DatabaseBuilder(db, options)
   .importCloudFormationDocs('data/CloudFormationDocumentation.json')
   .importStatefulResources('data/StatefulResources/StatefulResources.json')
   .importCannedMetrics('data/CloudWatchConsoleServiceDirectory.json'),
-  .importScrutinies()
-  .importAugmentations()
 
   // Apply the imports to the database
   .build();
@@ -36,18 +34,19 @@ new DatabaseBuilder(db, options)
 ### CLI
 
 ```console
-Usage: import-db [options] [db]
+Usage: import-db [options] [database]
 
 Import service specification sources into a service model database
 
 Arguments:
-  db                         The database file (default: "db.json")
+  database                         The database file (default: "db.json")
 
 Options:
-  -i, --input <db-file>      Load an existing database as base, imported sources are additive.
-  -c, --gzip                 Compress the database file using gzip
-  -f, --force                Force overwriting an existing file (default: false)
-  -d, --debug                Print additional debug output during import (default: false)
-  -r, --report <report-dir>  Create a detailed build report in the specified directory
-  -h, --help                 display help for command
+  -i, --input <database>           Load an existing database as base, imported sources are additive.
+  -c, --gzip                       Compress the database file using gzip
+  -f, --force                      Force overwriting an existing file (default: false)
+  -d, --debug                      Print additional debug output during import (default: false)
+  -r, --report <report-directory>  Create a detailed build report in the specified directory
+  -v, --validate                   Validate imported sources and fail if any data is invalid (default: false)
+  -h, --help                       display help for command
 ````

packages/@aws-cdk/service-spec-importers/src/cli/analyze-db.ts

@@ -1,4 +1,5 @@
 import { Property, RichPropertyType, RichSpecDatabase, SpecDatabase, loadDatabase } from '@aws-cdk/service-spec-types';
+import { handleFailure } from './util';
 
 async function main() {
   const db = await loadDatabase('db.json');
@@ -68,7 +69,4 @@ const ANALYZERS: Record<string, Analyzer> = {
   },
 };
 
-main().catch((e) => {
-  console.error(e);
-  process.exitCode = 1;
-});
+main().catch(handleFailure);

packages/@aws-cdk/service-spec-importers/src/cli/diff-db.ts

@@ -1,5 +1,6 @@
 import { loadDatabase } from '@aws-cdk/service-spec-types';
 import { Command } from 'commander';
+import { handleFailure } from './util';
 import { DbDiff } from '../db-diff';
 import { DiffFormatter } from '../diff-fmt';
 
@@ -36,7 +37,4 @@ async function main() {
   process.exitCode = hasChanges ? 1 : 0;
 }
 
-main().catch((e) => {
-  console.error(e);
-  process.exitCode = 1;
-});
+main().catch(handleFailure);