diff --git a/.projen/deps.json b/.projen/deps.json index 34feaa765..b2334b8c7 100644 --- a/.projen/deps.json +++ b/.projen/deps.json @@ -188,16 +188,13 @@ }, { "name": "jsii-docgen", + "version": "^10.2.0", "type": "build" }, { "name": "jsii-pacmak", "type": "build" }, - { - "name": "jsii-reflect", - "type": "build" - }, { "name": "jsii-rosetta", "type": "build" diff --git a/.projen/tasks.json b/.projen/tasks.json index 6fef622ef..b7f7831ac 100644 --- a/.projen/tasks.json +++ b/.projen/tasks.json @@ -1205,13 +1205,13 @@ }, "steps": [ { - "exec": "npx npm-check-updates@latest --upgrade --target=minor --peer --dep=dev --filter=@aws-cdk/aws-servicecatalogappregistry-alpha,@aws-sdk/client-sfn,@jsii/spec,@octokit/rest,@types/aws-lambda,@types/changelog-filename-regex,@types/fs-extra,@types/jest,@types/markdown-it,@types/markdown-it-emoji,@types/node,@types/semver,@types/streamx,@types/tar-stream,@types/tough-cookie,@types/uuid,@typescript-eslint/eslint-plugin,@typescript-eslint/parser,async-sema,aws-cdk-lib,aws-cdk,aws-embedded-metrics,aws-sdk,aws-sdk-mock,aws-xray-sdk-core,case,cdk-dia,cdk-watchful,cdklabs-projen-project-types,changelog-filename-regex,construct-hub-webapp,dotenv,esbuild,eslint-config-prettier,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,eslint,feed,fs-extra,glob,got,jest,jest-junit,jsii-diff,jsii-docgen,jsii-pacmak,jsii-reflect,jsii-rosetta,JSONStream,markdown-it,markdown-it-emoji,nock,normalize-registry-metadata,npm-check-updates,prettier,projen,semver,spdx-license-list,standard-version,streamx,tar-stream,ts-node,typescript,uuid,yaml" + "exec": "npx npm-check-updates@latest --upgrade --target=minor --peer --dep=dev --filter=@aws-cdk/aws-servicecatalogappregistry-alpha,@aws-sdk/client-sfn,@jsii/spec,@octokit/rest,@types/aws-lambda,@types/changelog-filename-regex,@types/fs-extra,@types/jest,@types/markdown-it,@types/markdown-it-emoji,@types/node,@types/semver,@types/streamx,@types/tar-stream,@types/tough-cookie,@types/uuid,@typescript-eslint/eslint-plugin,@typescript-eslint/parser,async-sema,aws-cdk-lib,aws-cdk,aws-embedded-metrics,aws-sdk,aws-sdk-mock,aws-xray-sdk-core,case,cdk-dia,cdk-watchful,cdklabs-projen-project-types,changelog-filename-regex,construct-hub-webapp,dotenv,esbuild,eslint-config-prettier,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,eslint,feed,fs-extra,glob,got,jest,jest-junit,jsii-diff,jsii-docgen,jsii-pacmak,jsii-rosetta,JSONStream,markdown-it,markdown-it-emoji,nock,normalize-registry-metadata,npm-check-updates,prettier,projen,semver,spdx-license-list,standard-version,streamx,tar-stream,ts-node,typescript,uuid,yaml" }, { "exec": "yarn install --check-files" }, { - "exec": "yarn upgrade @aws-cdk/aws-servicecatalogappregistry-alpha @aws-sdk/client-sfn @jsii/spec @octokit/rest @types/aws-lambda @types/changelog-filename-regex @types/fs-extra @types/jest @types/markdown-it @types/markdown-it-emoji @types/node @types/semver @types/streamx @types/tar-stream @types/tough-cookie @types/uuid @typescript-eslint/eslint-plugin @typescript-eslint/parser async-sema aws-cdk-lib aws-cdk aws-embedded-metrics aws-sdk aws-sdk-mock aws-xray-sdk-core case cdk-dia cdk-watchful cdklabs-projen-project-types changelog-filename-regex construct-hub-webapp dotenv esbuild eslint-config-prettier eslint-import-resolver-typescript eslint-plugin-import eslint-plugin-prettier eslint feed fs-extra glob got jest jest-junit jsii-diff jsii-docgen jsii-pacmak jsii-reflect jsii-rosetta JSONStream markdown-it markdown-it-emoji nock normalize-registry-metadata npm-check-updates prettier projen semver spdx-license-list standard-version streamx tar-stream ts-node typescript uuid yaml" + "exec": "yarn upgrade @aws-cdk/aws-servicecatalogappregistry-alpha @aws-sdk/client-sfn @jsii/spec @octokit/rest @types/aws-lambda @types/changelog-filename-regex @types/fs-extra @types/jest @types/markdown-it @types/markdown-it-emoji @types/node @types/semver @types/streamx @types/tar-stream @types/tough-cookie @types/uuid @typescript-eslint/eslint-plugin @typescript-eslint/parser async-sema aws-cdk-lib aws-cdk aws-embedded-metrics aws-sdk aws-sdk-mock aws-xray-sdk-core case cdk-dia cdk-watchful cdklabs-projen-project-types changelog-filename-regex construct-hub-webapp dotenv esbuild eslint-config-prettier eslint-import-resolver-typescript eslint-plugin-import eslint-plugin-prettier eslint feed fs-extra glob got jest jest-junit jsii-diff jsii-docgen jsii-pacmak jsii-rosetta JSONStream markdown-it markdown-it-emoji nock normalize-registry-metadata npm-check-updates prettier projen semver spdx-license-list standard-version streamx tar-stream ts-node typescript uuid yaml" }, { "exec": "npx projen" diff --git a/.projenrc.ts b/.projenrc.ts index 03eb00ae8..2fdf1cd1b 100644 --- a/.projenrc.ts +++ b/.projenrc.ts @@ -59,7 +59,6 @@ const project = new CdklabsConstructLibrary({ 'fs-extra', 'got', 'JSONStream', - 'jsii-reflect', 'semver', 'spdx-license-list', 'streamx', @@ -198,6 +197,10 @@ project.addTask('bundle', { description: 'Bundle all lambda and ECS functions', }); +// I have to put this here, otherwise projen overrides this with a `jsii-docgen` +// dependency without a version. I don't know where or why. +project.addDevDeps('jsii-docgen@^10.2.0'); + // extract the "build/" directory from "construct-hub-webapp" into "./website" // and bundle it with this library. this way, we are only taking a // dev-dependency on the webapp instead of a normal/bundled dependency. diff --git a/API.md b/API.md index ae106e6ae..d77ac852b 100644 --- a/API.md +++ b/API.md @@ -1,508 +1,3 @@ -# Construct Hub - -This project maintains a [AWS Cloud Development Kit][aws-cdk] construct library -that can be used to deploy instances of the Construct Hub in any AWS Account. - -This software backs the public instance of the -[ConstructHub](https://constructs.dev), and can be used to deploy a self-hosted -instance with personalized configuration. - -[aws-cdk]: https://github.com/aws/aws-cdk - -## :question: Getting Started - -> [!WARNING] -> -> ### Disclaimer -> -> The [public instance of ConstructHub](https://constructs.dev) is Generally Available. -> -> Self-hosted ConstructHub instances are however in active development and -> should be considered *experimental*. Breaking changes to the public API of -> this package are expected to be released without prior notice, and the -> infrastructure and operational posture of ConstructHub instances may also -> significantly change. -> -> You are welcome to deploy self-hosted instances of ConstructHub for evaluation -> purposes, and we welcome any feedback (good or bad) from your experience in -> doing so. - -> [!IMPORTANT] -> -> ### 💰 Cost of running Construct Hub -> -> If you opt to use Construct Hub for processing your CDK packages, -> you will be subject to charges based on the number of packages processed by Construct Hub. -> To minimize these charges, you can implement package filters for relevant sources -> and exclude public NPM packages from the processing list. - -### Quick Start - -Once you have installed the `construct-hub` library in your project, the -simplest way to get started is to create an instance of the `ConstructHub` -construct: - -```ts nofixture -import { App, Stack } from 'aws-cdk-lib/core'; -import { ConstructHub } from 'construct-hub'; - -// The usual... you might have used `cdk init app` instead! -const app = new App(); -const stack = new Stack(app, 'StackName', { /* ... */ }); - -// Now to business! -new ConstructHub(stack, 'ConstructHub'); -``` - -### Personalization - -#### Using a custom domain name - -In order to use a custom domain for your ConstructHub instance instead of the -default CloudFront domain name, specify the `domain` property with the following -elements: - -| Attribute | Description | -| ------------------------------ | ------------------------------------------------------------------------------------------------- | -| `zone` | A Route53 Hosted Zone, where DNS records will be added. | -| `cert` | An Amazon Certificate Manager certificate, which must be in the `us-east-1` region. | -| `monitorCertificateExpiration` | Set to `false` if you do not want an alarm to be created when the certificate is close to expiry. | - -Your self-hosted ConstructHub instance will be served from the root of the -provided `zone`, so the certificate must match this name. - -#### Alternate package sources - -By default, ConstructHub has a single package source configured: the public -`npmjs.com` registry. Self-hosted instances typically should list packages from -alternate sources, either in addition to packages from `npmjs.com`, or instead -of those. - -The `packageSources` property can be used to replace the default set of package -sources configured on the instance. ConstructHub provides `IPackageSource` -implementations for the public `npmjs.com` registry as well as for private -CodeArtifact repositories: - -```ts nofixture -import * as codeartifact from 'aws-cdk-lib/aws-codeartifact'; -import { App, Stack } from 'aws-cdk-lib/core'; -import { sources, ConstructHub } from 'construct-hub'; - -// The usual... you might have used `cdk init app` instead! -const app = new App(); -const stack = new Stack(app, 'StackName', { /* ... */ }); - -// Now to business! -const repository = new codeartifact.CfnRepository(stack, 'Repository', { - domainName: 'my-domain', - repositoryName: 'my-repo', - // .... -}); -new ConstructHub(stack, 'ConstructHub', { - packageSources: [ - new sources.NpmJs(), // Remove if you do NOT want npmjs.com packages - new sources.CodeArtifact({ repository }), - ], -}); -``` - -You may also implement a custom `IPackageSource` if you want to index packages -from alternate locations. In this case, the component you provide will be -responsible for sending notifications to an SQS Queue about newly discovered -packages. You may refer to the [sources.NpmJs] and [sources.CodeArtifact] -implementations as a reference for hos this can be done. - -By default, download counts of NPM packages will be fetched periodically from -NPM's public API by a Lambda. Since this is not desirable if you are using a -private package registry, this is automatically disabled if you specify your own -value for `packageSources`. (But this can be re-enabled through the -`fetchPackageStats` property if needed). - -[sources.NpmJs]: src/package-sources/npmjs.ts -[sources.CodeArtifact]: src/package-sources/code-artifact.ts - -#### Package deny list - -Certain packages may be undesirable to show in your self-hosted ConstructHub -instance. In order to prevent a package from ever being listed in construct hub, -the `denyList` property can be configured with a set of `DenyListRule` objects -that specify which package or package versions should never be lested: - -```ts nofixture -import { App, Stack } from 'aws-cdk-lib/core'; -import { ConstructHub } from 'construct-hub'; - -// The usual... you might have used `cdk init app` instead! -const app = new App(); -const stack = new Stack(app, 'StackName', { /* ... */ }); - -// Now to business! -new ConstructHub(stack, 'ConstructHub', { - denyList: [ - // Denying _all_ versions of the "sneaky-hackery" package - { packageName: 'sneaky-hackery', reason: 'Mines bitcoins wherever it gets installed' }, - // Denying _a specific_ version of the "bad-release" package - { packageName: 'bad-release', version: '1.2.3', reason: 'CVE-####-#####' }, - ], -}); -``` - -#### Redirecting from additional domains - -You can add additional domains that will be redirected to your primary Construct -Hub domain: - -```ts -import * as r53 from 'aws-cdk-lib/aws-route53'; - -const myDomainZone = r53.HostedZone.fromHostedZoneAttributes(this, 'MyDomainZone', { - hostedZoneId: 'AZ1234', - zoneName: 'my.domain.com', -}); - -new ConstructHub(this, 'ConstructHub', { - additionalDomains: [ { hostedZone: myDomainZone } ] -}); -``` - -This will set up full domain redirect using Amazon S3 and Amazon CloudFront. All -requests will be redirected to your primary Construct Hub domain. - -#### Decrease deployment footprint - -By default, ConstructHub executes the documentation rendering process in the -context of isolated subnets. This is a defense-in-depth mechanism to mitigate -the risks associated with downloading aribtrary (un-trusted) *npm packages* and -their dependency closures. - -This layer of security implies the creation of a number of resources that can -increase the operating cost of your self-hosted instance: several VPC endpoints -are created, an internal CodeArtifact repository needs to be provisioned, etc... - -While we generally recommend leaving these features enabled, if your self-hosted -ConstructHub instance only indexes *trusted* packages (as could be the case for -an instance that does not list packages from the public `npmjs.com` registry), -you may set the `isolateLambdas` setting to `false`. - -## :gear: Operating a self-hosted instance - -1. [Application Overview](./docs/application-overview.md) provides a high-level - description of the components that make a ConstructHub instance. This is a - great starting point for people who consider operating a self-hosted instance - of ConstructHub; and for new operators on-boarding the platform. - -1. [Operator Runbook](./docs/operator-runbook.md) is a series of diagnostics and - troubleshooting guides indended for operators to have a quick and easy way to - navigate a ConstructHub instance when they are reacting to an alarm or bug - report. - -### :baby_chick: Deployment Canaries - -Construct Hub provides several built-in validation mechanisms to make sure the -deployment of your instance is continuously operating as expected. - -These mechanisms come in the form of canary testers that are part of the -ConstructHub deployment stack. Each canary runs periodically and performs a -different check, triggering a different CloudWatch alarm in case it detects a -failure. - -We recommend that you use staged deployments, and block promotions to the -production stage in case any preivous stage triggers an alarm within a specific -timeframe. - -#### Discovery Canary - -When configuring an `NpmJs` package source, a package discovery canary can be -enabled using the `enableCanary` property (and optionally configured using the -`canaryPackage` and `canarySla` properties). This feature is activated by -default and monitors availability of releases of the `construct-hub-probe` npm -package in the ConstructHub instance. - -Probe packages, such as `construct-hub-probe` are published frequently (e.g: -every 3 hours or more frequently), and can be used to ensure the ConstructHub -instance correctly discovers, indexes and represents those packages. - -If a different package or SLA should be used, you can configure the `NpmJs` -package source manually like so: - -```ts nofixture -import * as codeartifact from 'aws-cdk-lib/aws-codeartifact'; -import { App, Duration, Stack } from 'aws-cdk-lib/core'; -import { sources, ConstructHub } from 'construct-hub'; - -const app = new App(); -const stack = new Stack(app, 'StackName', { /* ... */ }); - -new ConstructHub(stack, 'ConstructHub', { - // ... - packageSources: [ - // ... - new sources.NpmJs({ - enableCanary: true, // This is the default - canaryPackage: '@acme/my-constructhub-probe', - canarySla: Duration.minutes(30), - }), - // ... - ], - // ... -}); -``` - -In case the new package isn't fully available in the predefined SLA, a -**high severity** CloudWatch alarm will trigger, which will in turn trigger -the configured action for low severity alarms. - -> See [Monitoring & Alarms](./docs/application-overview.md#monitoring--alarming) - -The operator runbook contains [instructions](./docs/operator-runbook.md) on how -to diagnose and mitigate the root cause of the failure. - -### :nail_care: Customizing the frontend - -There are a number of customizations available in order to make your private -construct hub better tailored to your organization. - -#### Package Tags - -Configuring package tags allows you to compute additional labels to be applied -to packages. These can be used to indicate to users which packages are owned by -trusted organizations, or any other arbitrary conditions, and can be referenced -while searching. - -For example: - -```ts -new ConstructHub(this, "ConstructHub", { - packageTags: [{ - id: 'official', - condition: TagCondition.field('name').eq('construct-hub'), - keyword: { - label: 'Official', - color: '#00FF00', - }, - highlight: { - label: 'Vended by AWS', - color: '#00FF00', - } - }] -}); -``` - -The above example will result in packages with the `name` of `construct-hub` to -receive the `Official` tag, which is colored green and displayed amongst the -list of keywords. Additionally the `highlight` key shows this as a highlighted -item on the package's card. - -The `searchFilter` key can also be used to show tags as search filters grouped -together. - -```ts -const authorsGroup = new PackageTagGroup("authors", { - label: "Authors", - tooltip: "Information about the authors filter", - filterType: FilterType.checkbox(), -}); - -const isAws = TagCondition.field('name').eq('construct-hub'); -new ConstructHub(this, "ConstructHub", { - packageTags: [{ - id: 'AWS', - condition: isAws, - searchFilter: { - group: authorsGroup, - display: 'AWS', - }, - }, { - id: 'Community', - condition: TagCondition.not(isAws), - searchFilter: { - group: authorsGroup, - display: 'AWS', - }, - }] -}); -``` - -The above will show a list of `Authors` filters on the search results page -with a checkbox for each `AWS` and `Community` packages, allowing users to -filter results by the presence of these tags. - -Combinations of conditions are also supported: - -```ts -new ConstructHub(this, "ConstructHub", { - packageTags: [{ - id: 'official', - keyword: { - label: 'Official', - color: '#00FF00', - }, - condition: TagCondition.or( - TagCondition.field('name').eq('construct-hub'), - TagCondition.field('name').eq('construct-hub-webapp'), - ), - }] -}); -``` - -You can assert against any value within package json including nested ones. - -```ts -TagCondition.field('constructHub', 'nested', 'key').eq('value'); - -// checks the following: -// packageJson?.constructHub?.nested?.key === value; -``` - -You can also assert that a string occurs at least a certain number of times -within the package's README. - -```ts -TagCondition.readme().includes('ECS'); -TagCondition.readme().includes('fargate', { atLeast: 3, caseSensitive: false }); -``` - -#### Package Links - -Configuring package links allows you to replace the `Repository`, `License`, -and `Registry` links on the package details page with whatever you choose. - -For example: - -```ts -new ConstructHub(this, "ConstructHub", { - packageLinks: [{ - linkLabel: 'Service Level Agreement', - configKey: 'SLA', - }, { - linkLabel: 'Contact', - configKey: 'Contact', - linkText: 'Email Me!', - allowedDomains: ['me.com'], - }] -}); -``` - -This would allow publishers to add the following to their package.json: - -```json -"constructHub": { - "packageLinks": { - "SLA": "https://support.mypackage.com", - "Contact": "me.com/contact" - } -} -``` - -Then the links on the corresponding package page would show these items as -configured. - -### RSS/ATOM feeds for recent packages - -Construct hub automatically generates RSS/ATOM feed showing the list of latest -100 packages added. The generated feed can be configured to get release notes from -GitHub by configuring it with [personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token). -The access token has to be stored in AWS Secretsmanager and should be passed to `feedConfiguration` - -For example: - -```ts -new ConstructHub(this, "ConstructHub", { - feedConfiguration: { - githubTokenSecret: secretsmanager.Secret.fromSecretCompleteArn(this, 'GitHubToken', ''), - feedDescription: 'Latest Constructs in the construct hub', - feedTitle: 'Latest constructs', - } - } -); -``` - -#### Home Page - -The home page is divided into sections, each with a header and list of packages. -Currently, for a given section you can display either the most recently updated -packages, or a curated list of packages. - -For example: - -```ts -new ConstructHub(this, "ConstructHub", { - featuredPackages: { - sections: [ - { - name: "Recently updated", - showLastUpdated: 4 - }, - { - name: "From the AWS CDK", - showPackages: [ - { - name: "@aws-cdk/core" - }, - { - name: "@aws-cdk/aws-s3", - comment: "One of the most popular AWS CDK libraries!" - }, - { - name: "@aws-cdk/aws-lambda" - }, - { - name: "@aws-cdk/pipelines", - comment: "The pipelines L3 construct library abstracts away many of the details of managing software deployment within AWS." - } - ] - } - ] - } -}); -``` - -#### Browse Categories - -The Construct Hub home page includes a section that displays a set of buttons -that represent browsing categories (e.g. "Databases", "Monitoring", -"Serverless", etc). - -You can use the `categories` option to configure these categories. Each category -is defined by a `title` and a `url`, which will be the link associated with the -button. - -```ts -new ConstructHub(this, "ConstructHub", { - categories: [ - { title: 'Databases', url: '?keywords=databases' }, - { title: 'Monitoring', url: '?q=monitoring' }, - { title: 'Partners', url: '?tags=aws-partner' } - ] -}); -``` - -#### Feature Flags - -Feature flags for the web app can be used to enable or disable experimental -features. These can be customized through the `featureFlags` property - for -more information about the available flags, check the documentation for -. - -#### AppRegistry - -By default, an AppRegistry application will be created that is associated -with the stack you put the `ConstructHub` construct in. - -## :raised_hand: Contributing - -If you are looking to contribute to this project, but don't know where to start, -have a look at our [contributing guide](CONTRIBUTING.md)! - -## :cop: Security - -See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more -information. - -## :balance_scale: License - -This project is licensed under the Apache-2.0 License. - # API Reference ## Constructs diff --git a/package.json b/package.json index 7b8a508a1..490938f73 100644 --- a/package.json +++ b/package.json @@ -161,9 +161,8 @@ "jest-junit": "^15", "jsii": "5.1.x", "jsii-diff": "^1.91.0", - "jsii-docgen": "^9.2.2", + "jsii-docgen": "^10.2.0", "jsii-pacmak": "^1.91.0", - "jsii-reflect": "^1.91.0", "jsii-rosetta": "5.1.x", "JSONStream": "^1.3.5", "markdown-it": "^13.0.2", diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index 262254ac8..74f6df53b 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -2046,7 +2046,7 @@ def submit_response(event: dict, context, response_status: str, error_message: s "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "0d6a9d6251264bb8a7490049212d8459361e0e30fc0fd8e0dce8a6772b9edb69.zip", + "S3Key": "387331b895f8a4fa933088f63f69d924bfc4278c52bb07e738706d922bc5ec8f.zip", }, "Description": "Release note RSS feed updater", "Environment": { @@ -2273,7 +2273,7 @@ def submit_response(event: dict, context, response_status: str, error_message: s "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "2a8df31329d7d64eae00507571a2923da45e26b4e248ffe3ea65972aab09eb7f.zip", + "S3Key": "ec58244ed727ecbf74ebd49800ed7c6125501ff02083816a00487b963fbae3f8.zip", }, "Description": "[ConstructHub/Ingestion] Ingests new package versions into the Construct Hub", "Environment": { @@ -2699,7 +2699,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "056d86597ac082dec1dcabf17f239063c008171fb4ce593964c1c0a1a1a42ef1.zip", + "S3Key": "18fcaea7e0ff466e70bbb5829cb1be2ebf8766860dfb98f441d62d218134e953.zip", }, "Description": "[ConstructHub/Ingestion/ReIngest] The function used to reprocess packages through ingestion", "Environment": { @@ -3365,7 +3365,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "81ac80fadeb9d6944819df9b9f03bfa6cff84deaba38239ca0a8ef5c99addddf.zip", + "S3Key": "2f6c6997e80b2e8cb310e368acccd821106d8770a66864b833f2efb2b6107897.zip", }, "Description": "[ConstructHub/Inventory] A canary that periodically inspects the list of indexed packages", "Environment": { @@ -4851,7 +4851,7 @@ Direct link to function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "29f38e026379680fc93aa92d0efd788ccccdb417a496eb44155ac76d8a361248.zip", + "S3Key": "e5dc694579d534f3c5217900a1120dcda2d6f783dd575dae5fd39e490e737f24.zip", }, "Description": { "Fn::Join": [ @@ -5919,7 +5919,7 @@ Warning: State Machines executions that sent messages to the DLQ will not show a "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "8885ca3662897de4e45e398cc7b452cab4351d912b6f709617207d6c2624ae55.zip", + "S3Key": "c8c064526f8b092ec352234f04d89a4e21aa7df721bad47412d8f1593beca559.zip", }, "Description": "[ConstructHub/Orchestration/NeedsCatalogUpdate] Determines whether a package version requires a catalog update", "Environment": { @@ -6857,7 +6857,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:15f4d33fac5712848fb63756f7416b6841486695de00210ccb3d968c66e87f7a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:5c4d6f38566f4c2e887b00338121fa9fab1777dbaf4e34e30d2128bdcb1193b9", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -14940,7 +14940,7 @@ def submit_response(event: dict, context, response_status: str, error_message: s "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "0d6a9d6251264bb8a7490049212d8459361e0e30fc0fd8e0dce8a6772b9edb69.zip", + "S3Key": "387331b895f8a4fa933088f63f69d924bfc4278c52bb07e738706d922bc5ec8f.zip", }, "Description": "Release note RSS feed updater", "Environment": { @@ -15167,7 +15167,7 @@ def submit_response(event: dict, context, response_status: str, error_message: s "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "2a8df31329d7d64eae00507571a2923da45e26b4e248ffe3ea65972aab09eb7f.zip", + "S3Key": "ec58244ed727ecbf74ebd49800ed7c6125501ff02083816a00487b963fbae3f8.zip", }, "Description": "[ConstructHub/Ingestion] Ingests new package versions into the Construct Hub", "Environment": { @@ -15611,7 +15611,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "056d86597ac082dec1dcabf17f239063c008171fb4ce593964c1c0a1a1a42ef1.zip", + "S3Key": "18fcaea7e0ff466e70bbb5829cb1be2ebf8766860dfb98f441d62d218134e953.zip", }, "Description": "[ConstructHub/Ingestion/ReIngest] The function used to reprocess packages through ingestion", "Environment": { @@ -16357,7 +16357,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "81ac80fadeb9d6944819df9b9f03bfa6cff84deaba38239ca0a8ef5c99addddf.zip", + "S3Key": "2f6c6997e80b2e8cb310e368acccd821106d8770a66864b833f2efb2b6107897.zip", }, "Description": "[ConstructHub/Inventory] A canary that periodically inspects the list of indexed packages", "Environment": { @@ -17843,7 +17843,7 @@ Direct link to function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "29f38e026379680fc93aa92d0efd788ccccdb417a496eb44155ac76d8a361248.zip", + "S3Key": "e5dc694579d534f3c5217900a1120dcda2d6f783dd575dae5fd39e490e737f24.zip", }, "Description": { "Fn::Join": [ @@ -18911,7 +18911,7 @@ Warning: State Machines executions that sent messages to the DLQ will not show a "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "8885ca3662897de4e45e398cc7b452cab4351d912b6f709617207d6c2624ae55.zip", + "S3Key": "c8c064526f8b092ec352234f04d89a4e21aa7df721bad47412d8f1593beca559.zip", }, "Description": "[ConstructHub/Orchestration/NeedsCatalogUpdate] Determines whether a package version requires a catalog update", "Environment": { @@ -19876,7 +19876,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:15f4d33fac5712848fb63756f7416b6841486695de00210ccb3d968c66e87f7a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:5c4d6f38566f4c2e887b00338121fa9fab1777dbaf4e34e30d2128bdcb1193b9", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -27740,7 +27740,7 @@ def submit_response(event: dict, context, response_status: str, error_message: s "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "0d6a9d6251264bb8a7490049212d8459361e0e30fc0fd8e0dce8a6772b9edb69.zip", + "S3Key": "387331b895f8a4fa933088f63f69d924bfc4278c52bb07e738706d922bc5ec8f.zip", }, "Description": "Release note RSS feed updater", "Environment": { @@ -27967,7 +27967,7 @@ def submit_response(event: dict, context, response_status: str, error_message: s "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "2a8df31329d7d64eae00507571a2923da45e26b4e248ffe3ea65972aab09eb7f.zip", + "S3Key": "ec58244ed727ecbf74ebd49800ed7c6125501ff02083816a00487b963fbae3f8.zip", }, "Description": "[ConstructHub/Ingestion] Ingests new package versions into the Construct Hub", "Environment": { @@ -28393,7 +28393,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "056d86597ac082dec1dcabf17f239063c008171fb4ce593964c1c0a1a1a42ef1.zip", + "S3Key": "18fcaea7e0ff466e70bbb5829cb1be2ebf8766860dfb98f441d62d218134e953.zip", }, "Description": "[ConstructHub/Ingestion/ReIngest] The function used to reprocess packages through ingestion", "Environment": { @@ -29059,7 +29059,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "81ac80fadeb9d6944819df9b9f03bfa6cff84deaba38239ca0a8ef5c99addddf.zip", + "S3Key": "2f6c6997e80b2e8cb310e368acccd821106d8770a66864b833f2efb2b6107897.zip", }, "Description": "[ConstructHub/Inventory] A canary that periodically inspects the list of indexed packages", "Environment": { @@ -30545,7 +30545,7 @@ Direct link to function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "29f38e026379680fc93aa92d0efd788ccccdb417a496eb44155ac76d8a361248.zip", + "S3Key": "e5dc694579d534f3c5217900a1120dcda2d6f783dd575dae5fd39e490e737f24.zip", }, "Description": { "Fn::Join": [ @@ -31613,7 +31613,7 @@ Warning: State Machines executions that sent messages to the DLQ will not show a "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "8885ca3662897de4e45e398cc7b452cab4351d912b6f709617207d6c2624ae55.zip", + "S3Key": "c8c064526f8b092ec352234f04d89a4e21aa7df721bad47412d8f1593beca559.zip", }, "Description": "[ConstructHub/Orchestration/NeedsCatalogUpdate] Determines whether a package version requires a catalog update", "Environment": { @@ -32551,7 +32551,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:15f4d33fac5712848fb63756f7416b6841486695de00210ccb3d968c66e87f7a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:5c4d6f38566f4c2e887b00338121fa9fab1777dbaf4e34e30d2128bdcb1193b9", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -40517,7 +40517,7 @@ def submit_response(event: dict, context, response_status: str, error_message: s "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "0d6a9d6251264bb8a7490049212d8459361e0e30fc0fd8e0dce8a6772b9edb69.zip", + "S3Key": "387331b895f8a4fa933088f63f69d924bfc4278c52bb07e738706d922bc5ec8f.zip", }, "Description": "Release note RSS feed updater", "Environment": { @@ -40731,7 +40731,7 @@ def submit_response(event: dict, context, response_status: str, error_message: s "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "2a8df31329d7d64eae00507571a2923da45e26b4e248ffe3ea65972aab09eb7f.zip", + "S3Key": "ec58244ed727ecbf74ebd49800ed7c6125501ff02083816a00487b963fbae3f8.zip", }, "Description": "[ConstructHub/Ingestion] Ingests new package versions into the Construct Hub", "Environment": { @@ -41157,7 +41157,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "056d86597ac082dec1dcabf17f239063c008171fb4ce593964c1c0a1a1a42ef1.zip", + "S3Key": "18fcaea7e0ff466e70bbb5829cb1be2ebf8766860dfb98f441d62d218134e953.zip", }, "Description": "[ConstructHub/Ingestion/ReIngest] The function used to reprocess packages through ingestion", "Environment": { @@ -41823,7 +41823,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "81ac80fadeb9d6944819df9b9f03bfa6cff84deaba38239ca0a8ef5c99addddf.zip", + "S3Key": "2f6c6997e80b2e8cb310e368acccd821106d8770a66864b833f2efb2b6107897.zip", }, "Description": "[ConstructHub/Inventory] A canary that periodically inspects the list of indexed packages", "Environment": { @@ -43331,7 +43331,7 @@ Direct link to function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "29f38e026379680fc93aa92d0efd788ccccdb417a496eb44155ac76d8a361248.zip", + "S3Key": "e5dc694579d534f3c5217900a1120dcda2d6f783dd575dae5fd39e490e737f24.zip", }, "Description": { "Fn::Join": [ @@ -44399,7 +44399,7 @@ Warning: State Machines executions that sent messages to the DLQ will not show a "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "8885ca3662897de4e45e398cc7b452cab4351d912b6f709617207d6c2624ae55.zip", + "S3Key": "c8c064526f8b092ec352234f04d89a4e21aa7df721bad47412d8f1593beca559.zip", }, "Description": "[ConstructHub/Orchestration/NeedsCatalogUpdate] Determines whether a package version requires a catalog update", "Environment": { @@ -45337,7 +45337,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:15f4d33fac5712848fb63756f7416b6841486695de00210ccb3d968c66e87f7a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:5c4d6f38566f4c2e887b00338121fa9fab1777dbaf4e34e30d2128bdcb1193b9", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -53477,7 +53477,7 @@ def submit_response(event: dict, context, response_status: str, error_message: s "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "0d6a9d6251264bb8a7490049212d8459361e0e30fc0fd8e0dce8a6772b9edb69.zip", + "S3Key": "387331b895f8a4fa933088f63f69d924bfc4278c52bb07e738706d922bc5ec8f.zip", }, "Description": "Release note RSS feed updater", "Environment": { @@ -53704,7 +53704,7 @@ def submit_response(event: dict, context, response_status: str, error_message: s "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "2a8df31329d7d64eae00507571a2923da45e26b4e248ffe3ea65972aab09eb7f.zip", + "S3Key": "ec58244ed727ecbf74ebd49800ed7c6125501ff02083816a00487b963fbae3f8.zip", }, "Description": "[ConstructHub/Ingestion] Ingests new package versions into the Construct Hub", "Environment": { @@ -54130,7 +54130,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "056d86597ac082dec1dcabf17f239063c008171fb4ce593964c1c0a1a1a42ef1.zip", + "S3Key": "18fcaea7e0ff466e70bbb5829cb1be2ebf8766860dfb98f441d62d218134e953.zip", }, "Description": "[ConstructHub/Ingestion/ReIngest] The function used to reprocess packages through ingestion", "Environment": { @@ -54922,7 +54922,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "81ac80fadeb9d6944819df9b9f03bfa6cff84deaba38239ca0a8ef5c99addddf.zip", + "S3Key": "2f6c6997e80b2e8cb310e368acccd821106d8770a66864b833f2efb2b6107897.zip", }, "Description": "[ConstructHub/Inventory] A canary that periodically inspects the list of indexed packages", "Environment": { @@ -56437,7 +56437,7 @@ Direct link to function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "29f38e026379680fc93aa92d0efd788ccccdb417a496eb44155ac76d8a361248.zip", + "S3Key": "e5dc694579d534f3c5217900a1120dcda2d6f783dd575dae5fd39e490e737f24.zip", }, "Description": { "Fn::Join": [ @@ -57098,7 +57098,7 @@ Warning: State Machines executions that sent messages to the DLQ will not show a "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "8885ca3662897de4e45e398cc7b452cab4351d912b6f709617207d6c2624ae55.zip", + "S3Key": "c8c064526f8b092ec352234f04d89a4e21aa7df721bad47412d8f1593beca559.zip", }, "Description": "[ConstructHub/Orchestration/NeedsCatalogUpdate] Determines whether a package version requires a catalog update", "Environment": { @@ -58042,7 +58042,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:15f4d33fac5712848fb63756f7416b6841486695de00210ccb3d968c66e87f7a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:5c4d6f38566f4c2e887b00338121fa9fab1777dbaf4e34e30d2128bdcb1193b9", }, "LogConfiguration": { "LogDriver": "awslogs", diff --git a/src/__tests__/backend/transliterator/__snapshots__/index.test.ts.snap b/src/__tests__/backend/transliterator/__snapshots__/index.test.ts.snap index 40762c372..d8f6ce9e8 100644 --- a/src/__tests__/backend/transliterator/__snapshots__/index.test.ts.snap +++ b/src/__tests__/backend/transliterator/__snapshots__/index.test.ts.snap @@ -229,7 +229,7 @@ exports[`CodeArtifact repository 1`] = ` ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:15f4d33fac5712848fb63756f7416b6841486695de00210ccb3d968c66e87f7a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:5c4d6f38566f4c2e887b00338121fa9fab1777dbaf4e34e30d2128bdcb1193b9", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -2544,7 +2544,7 @@ exports[`VPC Endpoints 1`] = ` ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:15f4d33fac5712848fb63756f7416b6841486695de00210ccb3d968c66e87f7a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:5c4d6f38566f4c2e887b00338121fa9fab1777dbaf4e34e30d2128bdcb1193b9", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -5021,7 +5021,7 @@ exports[`VPC Endpoints and CodeArtifact repository 1`] = ` ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:15f4d33fac5712848fb63756f7416b6841486695de00210ccb3d968c66e87f7a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:5c4d6f38566f4c2e887b00338121fa9fab1777dbaf4e34e30d2128bdcb1193b9", }, "LogConfiguration": { "LogDriver": "awslogs", @@ -7332,7 +7332,7 @@ exports[`basic use 1`] = ` ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:15f4d33fac5712848fb63756f7416b6841486695de00210ccb3d968c66e87f7a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:5c4d6f38566f4c2e887b00338121fa9fab1777dbaf4e34e30d2128bdcb1193b9", }, "LogConfiguration": { "LogDriver": "awslogs", diff --git a/src/__tests__/backend/transliterator/transliterator.ecstask.test.ts b/src/__tests__/backend/transliterator/transliterator.ecstask.test.ts index e81d81ea9..917811437 100644 --- a/src/__tests__/backend/transliterator/transliterator.ecstask.test.ts +++ b/src/__tests__/backend/transliterator/transliterator.ecstask.test.ts @@ -15,7 +15,6 @@ import { MarkdownDocument } from 'jsii-docgen/lib/docgen/render/markdown-doc'; import { MarkdownRenderer } from 'jsii-docgen/lib/docgen/render/markdown-render'; import { Documentation } from 'jsii-docgen/lib/docgen/view/documentation'; -import { Submodule } from 'jsii-reflect'; import type { TransliteratorInput } from '../../../backend/payload-schema'; import { reset } from '../../../backend/shared/aws.lambda-shared'; import * as constants from '../../../backend/shared/constants'; @@ -94,10 +93,6 @@ describe('VPC Endpoints', () => { }; } } - - public async listSubmodules(): Promise { - return []; - } } forPackage.mockImplementation(async (_: string) => { @@ -234,9 +229,6 @@ test('corrupt assembly marker is uploaded for the necessary languages', async () public async toJson() { throw new CorruptedAssemblyError(); } - public async listSubmodules(): Promise { - return []; - } } forPackage.mockImplementation(async (_: string) => { @@ -294,9 +286,6 @@ test('corrupt assembly and uninstallable markers are deleted', async () => { public async toJson() { return new MarkdownDocument(); } - public async listSubmodules(): Promise { - return []; - } } forPackage.mockImplementation(async (_: string) => { @@ -403,29 +392,14 @@ test('uploads a file per language (scoped package)', async () => { }); test('uploads a file per submodule (unscoped package)', async () => { - // GIVEN - const assembly: spec.Assembly = { - targets: { python: {} }, - submodules: { - '@scope/package-name.sub1': {}, - '@scope/package-name.sub2': {}, - }, - } as any; - - const submodules: Submodule[] = Object.keys(assembly.submodules ?? {}).map( - (sm) => ({ name: sm.substring(sm.indexOf('.') + 1) } as any) - ); - // eslint-disable-next-line @typescript-eslint/no-require-imports const forPackage = require('jsii-docgen').Documentation .forPackage as jest.MockedFunction; forPackage.mockImplementation(async (target: string) => { - return new MockDocumentation( - target, - submodules - ) as unknown as Documentation; + return new MockDocumentation(target) as unknown as Documentation; }); + // GIVEN const packageName = 'package-name'; const packageVersion = '1.2.3-dev.4'; const event: TransliteratorInput = { @@ -441,6 +415,15 @@ test('uploads a file per submodule (unscoped package)', async () => { languages: { typescript: true }, }; + const assembly: spec.Assembly = { + targets: { python: {} }, + submodules: { + '@scope/package-name.sub1': {}, + '@scope/package-name.sub2': {}, + '@scope/package-name.sub2.nested': {}, + }, + } as any; + // mock the s3ObjectExists call mockHeadRequest('package.tgz'); @@ -452,15 +435,15 @@ test('uploads a file per submodule (unscoped package)', async () => { '/docs-typescript.md', '/docs-sub1-typescript.md', '/docs-sub2-typescript.md', + '/docs-sub2.nested-typescript.md', '/docs-typescript.json', '/docs-sub1-typescript.json', - '/docs-sub2-typescript.json' + '/docs-sub2-typescript.json', + '/docs-sub2.nested-typescript.json' ); - // WHEN const { created } = await handler(event); - // THEN expect(created).toEqual([ `data/${packageName}/v${packageVersion}/docs-typescript.json`, `data/${packageName}/v${packageVersion}/docs-typescript.md`, @@ -468,6 +451,8 @@ test('uploads a file per submodule (unscoped package)', async () => { `data/${packageName}/v${packageVersion}/docs-sub1-typescript.md`, `data/${packageName}/v${packageVersion}/docs-sub2-typescript.json`, `data/${packageName}/v${packageVersion}/docs-sub2-typescript.md`, + `data/${packageName}/v${packageVersion}/docs-sub2.nested-typescript.json`, + `data/${packageName}/v${packageVersion}/docs-sub2.nested-typescript.md`, ]); }); @@ -532,12 +517,6 @@ test.each([true, false])( describe('markers for un-supported languages', () => { test('uploads ".not-supported" markers as relevant', async () => { - // GIVEN - const assembly: spec.Assembly = { - targets: { phony: {} }, - submodules: { 'package-name.sub1': {}, 'package-name.sub2': {} }, - } as any; - // eslint-disable-next-line @typescript-eslint/no-require-imports const forPackage = require('jsii-docgen').Documentation .forPackage as jest.MockedFunction; @@ -546,17 +525,13 @@ describe('markers for un-supported languages', () => { public async toJson() { throw new LanguageNotSupportedError(); } - public async listSubmodules(): Promise { - return Object.keys(assembly.submodules ?? {}).map( - (sm) => ({ name: sm.substring(sm.indexOf('.') + 1) } as any) - ); - } } forPackage.mockImplementation(async (_: string) => { return new MockDocumentation() as unknown as Documentation; }); + // GIVEN const packageName = 'package-name'; const packageVersion = '1.2.3-dev.4'; @@ -574,6 +549,11 @@ describe('markers for un-supported languages', () => { languages: { python: true }, }; + const assembly: spec.Assembly = { + targets: { phony: {} }, + submodules: { 'package-name.sub1': {}, 'package-name.sub2': {} }, + } as any; + // mock the s3ObjectExists call mockHeadRequest('package.tgz'); @@ -604,18 +584,12 @@ describe('markers for un-supported languages', () => { }); class MockDocumentation { - public constructor( - private readonly target: string, - private readonly submodules?: Submodule[] - ) {} + public constructor(private readonly target: string) {} public async toJson() { return { render: () => `{ "content": "docs for ${this.target}" }`, }; } - public async listSubmodules(): Promise { - return this.submodules ?? []; - } } function mockFetchRequests(assembly: spec.Assembly, tarball: Buffer) { diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 456e9334e..0faaddd01 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -2510,7 +2510,7 @@ def submit_response(event: dict, context, response_status: str, error_message: s "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "0d6a9d6251264bb8a7490049212d8459361e0e30fc0fd8e0dce8a6772b9edb69.zip", + "S3Key": "387331b895f8a4fa933088f63f69d924bfc4278c52bb07e738706d922bc5ec8f.zip", }, "Description": "Release note RSS feed updater", "Environment": { @@ -2739,7 +2739,7 @@ def submit_response(event: dict, context, response_status: str, error_message: s "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "2a8df31329d7d64eae00507571a2923da45e26b4e248ffe3ea65972aab09eb7f.zip", + "S3Key": "ec58244ed727ecbf74ebd49800ed7c6125501ff02083816a00487b963fbae3f8.zip", }, "Description": "[ConstructHub/Ingestion] Ingests new package versions into the Construct Hub", "Environment": { @@ -3183,7 +3183,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "056d86597ac082dec1dcabf17f239063c008171fb4ce593964c1c0a1a1a42ef1.zip", + "S3Key": "18fcaea7e0ff466e70bbb5829cb1be2ebf8766860dfb98f441d62d218134e953.zip", }, "Description": "[ConstructHub/Ingestion/ReIngest] The function used to reprocess packages through ingestion", "Environment": { @@ -3942,7 +3942,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "81ac80fadeb9d6944819df9b9f03bfa6cff84deaba38239ca0a8ef5c99addddf.zip", + "S3Key": "2f6c6997e80b2e8cb310e368acccd821106d8770a66864b833f2efb2b6107897.zip", }, "Description": "[ConstructHub/Inventory] A canary that periodically inspects the list of indexed packages", "Environment": { @@ -5471,7 +5471,7 @@ Direct link to function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "29f38e026379680fc93aa92d0efd788ccccdb417a496eb44155ac76d8a361248.zip", + "S3Key": "e5dc694579d534f3c5217900a1120dcda2d6f783dd575dae5fd39e490e737f24.zip", }, "Description": { "Fn::Join": [ @@ -6142,7 +6142,7 @@ Warning: State Machines executions that sent messages to the DLQ will not show a "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "8885ca3662897de4e45e398cc7b452cab4351d912b6f709617207d6c2624ae55.zip", + "S3Key": "c8c064526f8b092ec352234f04d89a4e21aa7df721bad47412d8f1593beca559.zip", }, "Description": "[ConstructHub/Orchestration/NeedsCatalogUpdate] Determines whether a package version requires a catalog update", "Environment": { @@ -7134,7 +7134,7 @@ Warning: messages that resulted in a failed exectuion will NOT be in the DLQ!", ], "Essential": true, "Image": { - "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:15f4d33fac5712848fb63756f7416b6841486695de00210ccb3d968c66e87f7a", + "Fn::Sub": "\${AWS::AccountId}.dkr.ecr.\${AWS::Region}.\${AWS::URLSuffix}/cdk-hnb659fds-container-assets-\${AWS::AccountId}-\${AWS::Region}:5c4d6f38566f4c2e887b00338121fa9fab1777dbaf4e34e30d2128bdcb1193b9", }, "LogConfiguration": { "LogDriver": "awslogs", diff --git a/src/backend/shared/constants.ts b/src/backend/shared/constants.ts index 351c818c4..93dbf6266 100644 --- a/src/backend/shared/constants.ts +++ b/src/backend/shared/constants.ts @@ -123,12 +123,21 @@ export function getObjectKeys(packageName: string, packageVersion: string) { /** * The key suffix for documentation artifacts by language and submodule. + * + * SubmoduleFqn can cleverly be '*' which means "all submodules". */ export function docsKeySuffix( lang?: DocumentationLanguage | '*', - submodule?: string, + submoduleFqn?: string, fileExt?: string ) { + // We strip the first part (assembly name) off of the submodule name, but + // only if it's not '*' + const submodule = + submoduleFqn && submoduleFqn !== '*' + ? submoduleFqn.split('.').slice(1).join('.') + : submoduleFqn; + return `/docs-${submodule ? `${submodule}-` : ''}${lang}.${fileExt ?? '*'}`; } diff --git a/src/backend/transliterator/transliterator.ecstask.ts b/src/backend/transliterator/transliterator.ecstask.ts index 3ba91f96f..709338a56 100644 --- a/src/backend/transliterator/transliterator.ecstask.ts +++ b/src/backend/transliterator/transliterator.ecstask.ts @@ -93,10 +93,12 @@ export function handler( console.log(`Source Version: ${event.assembly.versionId}`); console.log(`Fetching assembly: ${event.assembly.key}`); - const assemblyResponse = await aws - .s3() - .getObject({ Bucket: event.bucket, Key: event.assembly.key }) - .promise(); + const assemblyResponse = await retry(() => + aws + .s3() + .getObject({ Bucket: event.bucket, Key: event.assembly.key }) + .promise() + ); if (!assemblyResponse.Body) { throw new Error( `Response body for assembly at key ${event.assembly.key} is empty` @@ -106,14 +108,12 @@ export function handler( const assembly = JSON.parse( assemblyResponse.Body.toString('utf-8') ) as Assembly; - const submodules = Object.keys(assembly.submodules ?? {}).map( - (s) => s.split('.')[1] - ); + const submoduleFqns = Object.keys(assembly.submodules ?? {}); console.log( - `Assembly ${assembly.name} has ${submodules.length} submodules.` + `Assembly ${assembly.name} has ${submoduleFqns.length} submodules.` ); - restrictSubmoduleCountToReturnable(event, submodules); + restrictSubmoduleCountToReturnable(event, submoduleFqns); console.log(`Fetching package: ${event.package.key}`); const tarballExists = await aws.s3ObjectExists( @@ -183,15 +183,15 @@ export function handler( metrics.setDimensions({}); metrics.setNamespace(METRICS_NAMESPACE); - async function renderAndDispatch(submodule?: string) { - const label = `Rendering documentation in ${lang} for ${packageFqn} (submodule: ${submodule})`; + async function renderAndDispatch(submoduleFqn?: string) { + const label = `Rendering documentation in ${lang} for ${packageFqn} (submodule: ${submoduleFqn})`; try { console.log(label); console.time(label); const docgenLang = docgen.Language.fromString(lang.name); const json = await docs.toJson({ - submodule, + submodule: submoduleFqn, language: docgenLang, }); @@ -214,7 +214,7 @@ export function handler( const jsonKey = formatArtifactKey( event.assembly, lang, - submodule, + submoduleFqn, 'json' ); console.log(`Uploading ${jsonKey}`); @@ -248,7 +248,7 @@ export function handler( const key = formatArtifactKey( event.assembly, lang, - submodule, + submoduleFqn, 'md' ); console.log(`Uploading ${key}`); @@ -263,24 +263,36 @@ export function handler( // if the package used to have a corrupt assembly, remove the marker for it. await unmarkPackage( - constants.corruptAssemblyKeySuffix(language, submodule, 'md') + constants.corruptAssemblyKeySuffix( + language, + submoduleFqn, + 'md' + ) ); } catch (e) { if (e instanceof docgen.LanguageNotSupportedError) { markPackage( e, - constants.notSupportedKeySuffix(language, submodule, 'json') + constants.notSupportedKeySuffix( + language, + submoduleFqn, + 'json' + ) ); markPackage( e, - constants.notSupportedKeySuffix(language, submodule, 'md') + constants.notSupportedKeySuffix( + language, + submoduleFqn, + 'md' + ) ); } else if (e instanceof docgen.CorruptedAssemblyError) { markPackage( e, constants.corruptAssemblyKeySuffix( language, - submodule, + submoduleFqn, 'json' ) ); @@ -288,7 +300,7 @@ export function handler( e, constants.corruptAssemblyKeySuffix( language, - submodule, + submoduleFqn, 'md' ) ); @@ -301,11 +313,8 @@ export function handler( } } await renderAndDispatch(); - const submoduleNames = (await docs.listSubmodules()).map( - (sm) => sm.name - ); - for (const submodule of submoduleNames) { - await renderAndDispatch(submodule); + for (const submoduleFqn of submoduleFqns) { + await renderAndDispatch(submoduleFqn); } } ); @@ -389,33 +398,36 @@ function uploadFile( : 'application/octet-stream'; const token = S3_SEMAPHORE.acquire(); - return aws - .s3() - .putObject({ - Bucket: bucket, - Key: key, - Body: body, - // We may not import anything that uses 'aws-cdk-lib' here - CacheControl: - 'public, max-age=300, must-revalidate, s-maxage=60, proxy-revalidate', - ContentEncoding: contentEncoding, - ContentType: contentType, - Metadata: { - 'Origin-Version-Id': sourceVersionId ?? 'N/A', - }, - }) - .promise() - .finally(() => S3_SEMAPHORE.release(token)); + return retry(() => + aws + .s3() + .putObject({ + Bucket: bucket, + Key: key, + Body: body, + // We may not import anything that uses 'aws-cdk-lib' here + CacheControl: + 'public, max-age=300, must-revalidate, s-maxage=60, proxy-revalidate', + ContentEncoding: contentEncoding, + ContentType: contentType, + Metadata: { + 'Origin-Version-Id': sourceVersionId ?? 'N/A', + }, + }) + .promise() + ).finally(() => S3_SEMAPHORE.release(token)); } function deleteFile(bucket: string, key: string) { - return aws - .s3() - .deleteObject({ - Bucket: bucket, - Key: key, - }) - .promise(); + return retry(() => + aws + .s3() + .deleteObject({ + Bucket: bucket, + Key: key, + }) + .promise() + ); } function anchorFormatter(type: JsiiEntity) { @@ -431,12 +443,12 @@ function anchorFormatter(type: JsiiEntity) { function formatArtifactKey( assemblyObject: S3ObjectVersion, lang: DocumentationLanguage, - submodule: string | undefined, + submoduleFqn: string | undefined, extension: string ) { return assemblyObject.key.replace( /\/[^/]+$/, - constants.docsKeySuffix(lang, submodule, extension) + constants.docsKeySuffix(lang, submoduleFqn, extension) ); } @@ -549,3 +561,32 @@ function restrictSubmoduleCountToReturnable( ); } } + +/** + * Retry a function a number of times if it happens to get throttled + */ +async function retry(cb: () => Promise): Promise { + const deadline = Date.now() + 30_000; // Half a minute minute + let sleepMs = 10; + while (true) { + try { + return await cb(); + } catch (e) { + if (!isRetryableError(e) || Date.now() >= deadline) { + throw e; + } + + await sleep(Math.floor(Math.random() * sleepMs)); + sleepMs *= 2; + } + } +} + +function isRetryableError(e: any) { + // Prepare for AWS SDK v3 already + return e.code === 'SlowDown' || e.name === 'SlowDown'; +} + +function sleep(ms: number): Promise { + return new Promise((ok) => setTimeout(ok, ms)); +} diff --git a/yarn.lock b/yarn.lock index 9ccba5afd..28b284a98 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3141,6 +3141,14 @@ chalk "^4.1.2" semver "^7.5.4" +"@jsii/check-node@1.92.0": + version "1.92.0" + resolved "https://registry.yarnpkg.com/@jsii/check-node/-/check-node-1.92.0.tgz#e05908d2c0875a728db14d73bb30459a73bd008e" + integrity sha512-MQnFvDIn/VOz4FzchobZ4dfrl6qfuZIlYviNbGXhPMSeJ92BVB2F+NEyem9Sg/Csy2ehhtO1FGaUj4mO9/7Ntg== + dependencies: + chalk "^4.1.2" + semver "^7.5.4" + "@jsii/spec@1.91.0", "@jsii/spec@^1.88.0", "@jsii/spec@^1.89.0", "@jsii/spec@^1.91.0": version "1.91.0" resolved "https://registry.yarnpkg.com/@jsii/spec/-/spec-1.91.0.tgz#91d624357a66148ab9a2d8b5fb26331fc7b01932" @@ -3155,6 +3163,13 @@ dependencies: ajv "^8.12.0" +"@jsii/spec@^1.92.0": + version "1.92.0" + resolved "https://registry.yarnpkg.com/@jsii/spec/-/spec-1.92.0.tgz#8fcd72a0ced63e83e72a010a499518d603f00198" + integrity sha512-6jbwQ2uCVOUq6eddKQG/cPzuUsdJwaszQstTZtruhhjWNuoC4CjT5eHlzjeBqtxQZpGiKkRRwPHb1bCEGgffxA== + dependencies: + ajv "^8.12.0" + "@leichtgewicht/ip-codec@^2.0.1": version "2.0.4" resolved "https://registry.yarnpkg.com/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz#b2ac626d6cb9c8718ab459166d4bb405b8ffa78b" @@ -10198,17 +10213,17 @@ jsii-diff@^1.91.0: log4js "^6.9.1" yargs "^16.2.0" -jsii-docgen@^9.2.2: - version "9.2.2" - resolved "https://registry.yarnpkg.com/jsii-docgen/-/jsii-docgen-9.2.2.tgz#de5dabdf0b8bf11cff2b60a787df55ac91af55f5" - integrity sha512-QoL0BCGZMU1TxqJ7/RyEVhdKGnE2ZQnay9A35yJ8jkb31qyl6HN8dnAzfrz1ym4VbYtwlSqQ+Occjor8E/uxow== +jsii-docgen@^10.2.0: + version "10.2.0" + resolved "https://registry.yarnpkg.com/jsii-docgen/-/jsii-docgen-10.2.0.tgz#e43ac2bb7bc2e1dfb6a0847fc353ed9c3481d3dc" + integrity sha512-5iCDzjBV/LChgXCnsTHHJP8mbd9bFEoxqVPF6MuvzZ09f2XV0GCUoC+4Bua0Ck3oLsCpk+Z4dfTqTq/O2qwnPg== dependencies: - "@jsii/spec" "^1.89.0" + "@jsii/spec" "^1.92.0" case "^1.6.3" fs-extra "^10.1.0" glob "^8.1.0" glob-promise "^6.0.5" - jsii-reflect "^1.89.0" + jsii-reflect "^1.92.0" semver "^7.5.4" yargs "^16.2.0" @@ -10243,6 +10258,18 @@ jsii-reflect@^1.89.0, jsii-reflect@^1.91.0: oo-ascii-tree "^1.91.0" yargs "^16.2.0" +jsii-reflect@^1.92.0: + version "1.92.0" + resolved "https://registry.yarnpkg.com/jsii-reflect/-/jsii-reflect-1.92.0.tgz#d6d29b01b40560f7a6b2400646410688543856b5" + integrity sha512-9ZLaygpYXG7UBqcUtgINd4pDL43l2gdUwOqWUrMz5beVOgp1ccvbihVPW6LtWGcTDfC+A2oNzjEP6roUvWvDDw== + dependencies: + "@jsii/check-node" "1.92.0" + "@jsii/spec" "^1.92.0" + chalk "^4" + fs-extra "^10.1.0" + oo-ascii-tree "^1.92.0" + yargs "^16.2.0" + jsii-rosetta@5.1.x: version "5.1.13" resolved "https://registry.yarnpkg.com/jsii-rosetta/-/jsii-rosetta-5.1.13.tgz#5f5ccf112000c573747d039725d3cf48378ef781" @@ -11431,6 +11458,11 @@ oo-ascii-tree@^1.91.0: resolved "https://registry.yarnpkg.com/oo-ascii-tree/-/oo-ascii-tree-1.91.0.tgz#3add7a82988966b04ac522c74ad7dccbb15bc5cf" integrity sha512-pH4FFNLTO8VG+vNUm4AHmhLNjoSfQlO1UKIrzBGdeScppY785D93GVMH1XkCwbuO7pd9rrh6El5WF3a5QNvaww== +oo-ascii-tree@^1.92.0: + version "1.92.0" + resolved "https://registry.yarnpkg.com/oo-ascii-tree/-/oo-ascii-tree-1.92.0.tgz#9d9fcc1b8c6e8b5a79b8e64f8a8143ff59945d77" + integrity sha512-rLSPbnakn5Wb3dOIVtrmn8jfHKqWv7bROpyBiw6DExq+dOG7qC49EIs89hBhyHkvLolX0oC+0a/RMPAyHEZ+1w== + open@^8.0.9, open@^8.4.0: version "8.4.2" resolved "https://registry.yarnpkg.com/open/-/open-8.4.2.tgz#5b5ffe2a8f793dcd2aad73e550cb87b59cb084f9"