diff --git a/src/docs/PlatformDocumentationGenerator.ts b/src/docs/PlatformDocumentationGenerator.ts index 0643317..5a45ccc 100644 --- a/src/docs/PlatformDocumentationGenerator.ts +++ b/src/docs/PlatformDocumentationGenerator.ts @@ -27,7 +27,7 @@ export class PlatformDocumentationGenerator { private readonly kitDependencyAnalyzer: KitDependencyAnalyzer, private readonly controls: ComplianceControlRepository, private readonly terragrunt: TerragruntCliFacade, - private readonly logger: Logger, + private readonly logger: Logger ) {} async generate(docsRepo: DocumentationRepository) { @@ -40,24 +40,24 @@ export class PlatformDocumentationGenerator { const dest = docsRepo.resolvePlatformsPath("README.md"); this.logger.verbose( - (fmt) => `Copying ${fmt.kitPath(source)} to ${fmt.kitPath(dest)}`, + (fmt) => `Copying ${fmt.kitPath(source)} to ${fmt.kitPath(dest)}` ); await fs.ensureDir(path.dirname(dest)); await fs.copy(source, dest, { overwrite: true }); } private async generatePlatformsDocumentation( - docsRepo: DocumentationRepository, + docsRepo: DocumentationRepository ) { const foundationProgress = new ProgressReporter( "generate documentation", this.repo.relativePath(this.foundation.resolvePath()), - this.logger, + this.logger ); - const foundationDependencies = await this.kitDependencyAnalyzer - .findKitModuleDependencies( - this.foundation, + const foundationDependencies = + await this.kitDependencyAnalyzer.findKitModuleDependencies( + this.foundation ); for (const p of foundationDependencies.platforms) { @@ -69,19 +69,23 @@ export class PlatformDocumentationGenerator { private async generatePlatforDocumentation( dependencies: PlatformDependencies, - docsRepo: DocumentationRepository, + docsRepo: DocumentationRepository ) { const platformPath = this.foundation.resolvePlatformPath( - dependencies.platform, + dependencies.platform ); const platformProgress = new ProgressReporter( "generate documentation", this.repo.relativePath(platformPath), - this.logger, + this.logger ); - const platformModuleDocumentation = await this - .buildPlatformModuleOutputCollector(dependencies.platform); + const platformModuleDocumentation = + new RunIndividualPlatformModuleOutputCollector( + this.repo, + this.terragrunt, + this.logger + ); // as a fallback process modules serially, unfortunately this is the only "safe" way to collect output // see https://github.com/meshcloud/collie-cli/issues/265 @@ -92,74 +96,22 @@ export class PlatformDocumentationGenerator { dep, documentationMd, docsRepo, - dependencies.platform, + dependencies.platform ); } platformProgress.done(); } - private async buildPlatformModuleOutputCollector(platform: PlatformConfig) { - const platformHclPath = this.foundation.resolvePlatformPath( - platform, - "platform.hcl", - ); - - const platformHcl = await Deno.readTextFile(platformHclPath); - - const fastModeIdentifier = - "--- BEGIN COLLIE PLATFORM MODULE OUTPUT: ${path_relative_to_include()} ---"; - - this.logger.verbose( - (fmt) => - `detecting if fast output collection is supported in ${ - fmt.kitPath( - platformHclPath, - ) - } by looking for a before_hook emitting "${fastModeIdentifier}"`, - ); - const enableFastMode = platformHcl.includes(fastModeIdentifier); - this.logger.verbose( - (_) => "fast output collection is supported: " + enableFastMode, - ); - - if (enableFastMode) { - const platformPath = this.foundation.resolvePlatformPath(platform); - const collector = new RunAllPlatformModuleOutputCollector( - this.terragrunt, - this.logger, - ); - - await collector.initialize(platformPath); - - return collector; - } else { - this.logger.tip( - (f) => - `Enable fast output collection for collie in ${ - f.kitPath( - platformHclPath, - ) - }`, - ); - - return new RunIndividualPlatformModuleOutputCollector( - this.repo, - this.terragrunt, - this.logger, - ); - } - } - private async generatePlatformModuleDocumentation( dep: KitModuleDependency, documentationMd: string, docsRepo: DocumentationRepository, - platform: PlatformConfig, + platform: PlatformConfig ) { const destPath = docsRepo.resolvePlatformModulePath( platform.id, - dep.kitModuleId, + dep.kitModuleId ); await fs.ensureDir(path.dirname(destPath)); // todo: should we do nesting in the docs output or "flatten" module prefixes? @@ -169,14 +121,14 @@ export class PlatformDocumentationGenerator { const complianceStatementsBlock = this.generateComplianceStatementSection( dep, docsRepo, - destPath, + destPath ); mdSections.push(complianceStatementsBlock); const kitModuleSection = this.generateKitModuleSection( dep, docsRepo, - destPath, + destPath ); mdSections.push(kitModuleSection); @@ -184,30 +136,28 @@ export class PlatformDocumentationGenerator { this.logger.verbose( (fmt) => - `Wrote output "documentation_md" from platform module ${ - fmt.kitPath( - dep.sourcePath, - ) - } to ${fmt.kitPath(destPath)}`, + `Wrote output "documentation_md" from platform module ${fmt.kitPath( + dep.sourcePath + )} to ${fmt.kitPath(destPath)}` ); } private generateKitModuleSection( dep: KitModuleDependency, docsRepo: DocumentationRepository, - destPath: string, + destPath: string ) { if (!dep.kitModule) { return MarkdownUtils.container( "warning", "Invalid Kit Module Dependency", - "Could not find kit module at " + MarkdownUtils.code(dep.kitModulePath), + "Could not find kit module at " + MarkdownUtils.code(dep.kitModulePath) ); } const kitModuleLink = MarkdownUtils.link( dep.kitModule.name + " kit module", - docsRepo.kitModuleLink(destPath, dep.kitModuleId), + docsRepo.kitModuleLink(destPath, dep.kitModuleId) ); const kitModuleSection = `::: tip Kit module @@ -219,25 +169,23 @@ This platform module is a deployment of kit module ${kitModuleLink}. private generateComplianceStatementSection( dep: KitModuleDependency, docsRepo: DocumentationRepository, - destPath: string, + destPath: string ) { const complianceStatements = dep?.kitModule?.compliance ?.map((x) => { const control = this.controls.tryFindById(x.control); if (!control) { this.logger.warn( - `could not find compliance control ${x.control} referenced in a compliance statement in ${dep.kitModulePath}`, + `could not find compliance control ${x.control} referenced in a compliance statement in ${dep.kitModulePath}` ); return; } - return `- [${control.name}](${ - docsRepo.controlLink( - destPath, - x.control, - ) - }): ${x.statement}`; + return `- [${control.name}](${docsRepo.controlLink( + destPath, + x.control + )}): ${x.statement}`; }) .filter((x): x is string => !!x); diff --git a/src/docs/PlatformModuleOutputCollector.ts b/src/docs/PlatformModuleOutputCollector.ts index 94396b0..96c0fe2 100644 --- a/src/docs/PlatformModuleOutputCollector.ts +++ b/src/docs/PlatformModuleOutputCollector.ts @@ -6,82 +6,28 @@ import { CollieRepository } from "../model/CollieRepository.ts"; import { Logger } from "../cli/Logger.ts"; import { MeshError } from "../errors.ts"; -export interface PlatformModuleOutputCollector { - getOutput(dep: KitModuleDependency): Promise; -} - -interface PlatformModuleOutput { - module: string; - output: string; -} - -export class RunAllPlatformModuleOutputCollector - implements PlatformModuleOutputCollector { - private modules: PlatformModuleOutput[] = []; - - constructor( - private readonly terragrunt: TerragruntCliFacade, - private readonly logger: Logger, - ) {} - - async initialize(platformPath: string) { - const result = await this.terragrunt.collectOutputs( - platformPath, - "documentation_md", - ); - - if (!result.status.success) { - this.logger.error( - (fmt) => - `Failed to collect output "documentation_md" from platform ${ - fmt.kitPath( - platformPath, - ) - }`, - ); - this.logger.error(result.stderr); - - throw new MeshError( - "Failed to collect documentation output from platform modules", - ); - } - - this.modules = RunAllPlatformModuleOutputCollector.parseTerragrunt( - result.stdout, - ); - } - - // deno-lint-ignore require-await - async getOutput(dep: KitModuleDependency): Promise { - const platformModulePath = path.dirname(dep.sourcePath); - const module = this.modules.find((x) => - platformModulePath.endsWith(x.module) - ); - if (!module) { - throw new MeshError( - "Failed to find documentation output from platform module " + - platformModulePath, - ); - } - return module?.output; - } +/** + * Note: + * For a great UX/DX it's important that running "collie foundation docs" is fast. + * + * We have therefore tried speeding it up by collecting output from platform modules in parallel. + * Unfortunately, it appears that terragrunt does not offer us a good way to reliably get all the outputs from all + * platform modules, see https://github.com/meshcloud/collie-cli/issues/267 + * + * This "fast mode" detection also caused other bugs like https://github.com/meshcloud/collie-cli/issues/269 + * + * In the future, we should maybe investigate cachingas an alternative to parallelization, because usually an engineer + * would re-run "collie foundation docs" only after changing a specific platform module + */ - public static parseTerragrunt(stdout: string): PlatformModuleOutput[] { - const regex = /^--- BEGIN COLLIE PLATFORM MODULE OUTPUT: (.+) ---$/gm; - - const sections = stdout.split(regex); - const cleanedSections = sections - .map((x) => x.trim()) - .filter((x) => x !== ""); - - return chunk(cleanedSections, 2).map(([module, output]) => ({ - module, - output, - })); - } +export interface PlatformModuleOutputCollector { + getOutput(dep: KitModuleDependency): Promise; } +/** + * Collects platform module output by running each platform module individually in series. + */ export class RunIndividualPlatformModuleOutputCollector implements PlatformModuleOutputCollector { constructor(