From 0d97dda762e1ce76f47f1006ae8e6f559d461d16 Mon Sep 17 00:00:00 2001 From: Michiel De Smet Date: Thu, 24 Oct 2024 17:57:35 +0800 Subject: [PATCH 1/6] Get project name from dbt (#1469) --- src/dbt_client/dbtCloudIntegration.ts | 15 +++++++++++++++ src/dbt_client/dbtCoreIntegration.ts | 12 ++++++++++++ src/dbt_client/dbtIntegration.ts | 1 + src/manifest/dbtProject.ts | 2 +- 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/dbt_client/dbtCloudIntegration.ts b/src/dbt_client/dbtCloudIntegration.ts index 1c696cadc..5c748d5fd 100644 --- a/src/dbt_client/dbtCloudIntegration.ts +++ b/src/dbt_client/dbtCloudIntegration.ts @@ -174,6 +174,7 @@ export class DBTCloudProjectIntegration private static QUEUE_ALL = "all"; private targetPath?: string; private version: number[] | undefined; + private projectName: string = "unknown_" + crypto.randomUUID(); private adapterType: string = "unknown"; private packagesInstallPath?: string; private modelPaths?: string[]; @@ -376,6 +377,10 @@ export class DBTCloudProjectIntegration return this.version || [0, 0, 0]; } + getProjectName(): string { + return this.projectName; + } + getPythonBridgeStatus(): boolean { return this.python.connected; } @@ -988,6 +993,15 @@ export class DBTCloudProjectIntegration stderr, ); } + const lookupValue = (lookupString: string) => { + const regexString = `${lookupString}\\s*(.*)`; + const regexp = new RegExp(regexString, "gm"); + const matches = regexp.exec(stdout); + if (matches?.length === 2) { + return matches[1]; + } + throw new Error(`Could not find any entries for ${lookupString}`); + }; const lookupEntries = (lookupString: string) => { const regexString = `${lookupString}\\s*\\[(.*)\\]`; const regexp = new RegExp(regexString, "gm"); @@ -1007,6 +1021,7 @@ export class DBTCloudProjectIntegration this.macroPaths = lookupEntries("Macro paths").map((p) => join(this.projectRoot.fsPath, p), ); + this.projectName = lookupValue("Project name"); this.packagesInstallPath = join(this.projectRoot.fsPath, "dbt_packages"); } catch (error) { this.terminal.warn( diff --git a/src/dbt_client/dbtCoreIntegration.ts b/src/dbt_client/dbtCoreIntegration.ts index 212a2d8ad..b5be8f9ad 100644 --- a/src/dbt_client/dbtCoreIntegration.ts +++ b/src/dbt_client/dbtCoreIntegration.ts @@ -243,6 +243,7 @@ export class DBTCoreProjectIntegration private adapterType?: string; private targetName?: string; private version?: number[]; + private projectName: string = "unknown_" + crypto.randomUUID(); private packagesInstallPath?: string; private modelPaths?: string[]; private seedPaths?: string[]; @@ -378,6 +379,7 @@ export class DBTCoreProjectIntegration this.macroPaths = await this.findMacroPaths(); this.packagesInstallPath = await this.findPackagesInstallPath(); this.version = await this.findVersion(); + this.projectName = await this.findProjectName(); this.adapterType = await this.findAdapterType(); } @@ -560,6 +562,10 @@ export class DBTCoreProjectIntegration return this.version; } + getProjectName(): string { + return this.projectName; + } + async findAdapterType(): Promise { return this.python.lock( (python) => python`project.config.credentials.type`, @@ -1060,6 +1066,12 @@ export class DBTCoreProjectIntegration ); } + private async findProjectName(): Promise { + return this.python?.lock( + (python) => python!`to_dict(project.config.project_name)`, + ); + } + private throwBridgeErrorIfAvailable() { const allDiagnostics: DiagnosticCollection[] = [ this.pythonBridgeDiagnostics, diff --git a/src/dbt_client/dbtIntegration.ts b/src/dbt_client/dbtIntegration.ts index 88cddc110..ca6ea18ac 100644 --- a/src/dbt_client/dbtIntegration.ts +++ b/src/dbt_client/dbtIntegration.ts @@ -336,6 +336,7 @@ export interface DBTProjectIntegration extends Disposable { getPackageInstallPath(): string | undefined; getAdapterType(): string | undefined; getVersion(): number[] | undefined; + getProjectName(): string; getSelectedTarget(): string | undefined; // parse manifest rebuildManifest(): Promise; diff --git a/src/manifest/dbtProject.ts b/src/manifest/dbtProject.ts index 35115348a..f9ff618e3 100644 --- a/src/manifest/dbtProject.ts +++ b/src/manifest/dbtProject.ts @@ -207,7 +207,7 @@ export class DBTProject implements Disposable { } public getProjectName() { - return this.projectConfig.name; + return this.dbtProjectIntegration.getProjectName(); } getSelectedTarget() { From 0078e1c39b6558e148e1fcb893b8e2b879a6a951 Mon Sep 17 00:00:00 2001 From: Michiel De Smet Date: Thu, 24 Oct 2024 20:55:20 +0800 Subject: [PATCH 2/6] Fix target names lookup (#1477) Co-authored-by: anandgupta42 <93243293+anandgupta42@users.noreply.github.com> --- dbt_core_integration.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt_core_integration.py b/dbt_core_integration.py index 0625cc840..cdcc0fc0c 100644 --- a/dbt_core_integration.py +++ b/dbt_core_integration.py @@ -869,9 +869,9 @@ def validate_sql_dry_run(self, compiled_sql: str): def get_target_names(self): from dbt.config.profile import read_profile profile = read_profile(self.args.profiles_dir) - project = profile[self.project_name] - if "outputs" in project: - outputs = project["outputs"] + profile = profile[self.config.profile_name] + if "outputs" in profile: + outputs = profile["outputs"] return outputs.keys() return [] From 80bf682fd67d149eeb3d6f366e5c0f50095733d1 Mon Sep 17 00:00:00 2001 From: Saravanan Date: Fri, 25 Oct 2024 08:31:53 +0530 Subject: [PATCH 3/6] fix: enable teammates in vscode by default (#1479) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d3c784622..de10e91f8 100644 --- a/package.json +++ b/package.json @@ -100,7 +100,7 @@ "dbt.enableTeammates": { "type": "boolean", "description": "Enable AltimateAI teammates feature.", - "default": false + "default": true }, "dbt.disableQueryHistory": { "type": "boolean", From 16d272f1e7ed894e62c9f955ed3dce8bf4282d39 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Thu, 24 Oct 2024 21:59:21 -0700 Subject: [PATCH 4/6] bumping version 0.47.4 -> 0.47.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index de10e91f8..98cca6dcc 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "bugs": { "url": "https://github.com/AltimateAI/vscode-dbt-power-user/issues" }, - "version": "0.47.4", + "version": "0.47.5", "engines": { "vscode": "^1.81.0" }, From 1654373bc2c637811eb887bb362b30f3e319ee32 Mon Sep 17 00:00:00 2001 From: Saravanan Date: Fri, 25 Oct 2024 10:43:15 +0530 Subject: [PATCH 5/6] fix: preserve comments and formatting in yml (#1468) * fix: preserve comments and formatting in yml * fix: schema save * chore: rename model * fix: doc edit save use cases * fix: formatting issue --- src/services/docGenService.ts | 18 ++ .../altimateWebviewProvider.ts | 21 +- src/webview_provider/docsEditPanel.ts | 250 ++++++++++++------ 3 files changed, 192 insertions(+), 97 deletions(-) diff --git a/src/services/docGenService.ts b/src/services/docGenService.ts index c13294c46..7f924299c 100644 --- a/src/services/docGenService.ts +++ b/src/services/docGenService.ts @@ -22,6 +22,24 @@ import { import { QueryManifestService } from "./queryManifestService"; import { TelemetryEvents } from "../telemetry/events"; +export interface DocumentationSchemaColumn { + name: string; + description: string; + data_type?: string; + quote?: boolean; + [key: string]: unknown; +} +interface DocumentationSchemaModel { + name: string; + description: string; + tests: any; + columns: []; +} +export interface DocumentationSchema { + version: number; + models: DocumentationSchemaModel[]; +} + interface GenerateDocsForColumnsProps { panel: WebviewView | WebviewPanel | undefined; message: any; diff --git a/src/webview_provider/altimateWebviewProvider.ts b/src/webview_provider/altimateWebviewProvider.ts index 5aa31e464..19013d653 100644 --- a/src/webview_provider/altimateWebviewProvider.ts +++ b/src/webview_provider/altimateWebviewProvider.ts @@ -243,17 +243,16 @@ export class AltimateWebviewProvider implements WebviewViewProvider { try { switch (command) { - case "getTeammatesStatus": - { - const isEnabled = workspace - .getConfiguration("dbt") - .get("enableTeammates", false); - this.sendResponseToWebview({ - command: "teammatesUpdated", - data: isEnabled, - }); - break; - } + case "getTeammatesStatus": { + const isEnabled = workspace + .getConfiguration("dbt") + .get("enableTeammates", false); + this.sendResponseToWebview({ + command: "teammatesUpdated", + data: isEnabled, + }); + break; + } case "configEnabled": this.handleSyncRequestFromWebview( syncRequestId, diff --git a/src/webview_provider/docsEditPanel.ts b/src/webview_provider/docsEditPanel.ts index 6fba0d419..15e797da5 100644 --- a/src/webview_provider/docsEditPanel.ts +++ b/src/webview_provider/docsEditPanel.ts @@ -32,10 +32,14 @@ import path = require("path"); import { PythonException } from "python-bridge"; import { TelemetryService } from "../telemetry"; import { AltimateRequest } from "../altimate"; -import { stringify, parse } from "yaml"; +import { stringify, parse, parseDocument, YAMLSeq, YAMLMap } from "yaml"; import { NewDocsGenPanel } from "./newDocsGenPanel"; import { DBTProject } from "../manifest/dbtProject"; -import { DocGenService } from "../services/docGenService"; +import { + DocGenService, + DocumentationSchema, + DocumentationSchemaColumn, +} from "../services/docGenService"; import { DBTTerminal } from "../dbt_client/dbtTerminal"; import { TestMetaData, @@ -465,6 +469,46 @@ export class DocsEditViewPanel implements WebviewViewProvider { return this.modifyColumnNames(columns, existingColumnNames); } + private setOrDeleteInParsedDocument( + doc: YAMLMap, + key: string, + value: any, + ) { + if (value) { + doc.set(key, value); + } else { + doc.delete(key); + } + } + + private findEntityInParsedDoc( + models: + | YAMLSeq + | YAMLSeq + | undefined, + predicate: (name: string) => boolean, + ) { + if (models && models.items) { + return ( + // @ts-ignore + (models.items.find( + ( + item: + | DocumentationSchema["models"]["0"] + | DocumentationSchemaColumn, + ) => { + if (item instanceof YAMLMap) { + const name = item.get("name"); + return name && predicate(name as string); + } + return false; + }, + ) as YAMLMap | undefined) || null + ); + } + return null; + } + private setupWebviewHooks(context: WebviewViewResolveContext) { // Clear this listener before subscribing again if (this.onMessageDisposable) { @@ -672,33 +716,125 @@ export class DocsEditViewPanel implements WebviewViewProvider { const docFile: string = readFileSync(patchPath).toString("utf8"); - const parsedDocFile = - parse(docFile, { - strict: false, - uniqueKeys: false, - maxAliasCount: -1, - }) || {}; - if (parsedDocFile.models === undefined) { - // this is a fresh file or one without models, so init the models - parsedDocFile.models = []; - } - if ( - parsedDocFile.models.find( - (model: any) => model.name === message.name, - ) === undefined - ) { + const parsedDocFile = parseDocument< + YAMLSeq + >(docFile, { + strict: false, + uniqueKeys: false, + }); + const existingModels = parsedDocFile.get("models") as + | YAMLSeq + | undefined; + + const model = this.findEntityInParsedDoc( + existingModels, + (name: string) => name === message.name, + ); + + if (!model) { // there is a models section but the model does not exist yet. - parsedDocFile.models.push({ + const newModelData = { name: message.name, - description: message.description || undefined, - columns: message.columns.map((column: any) => { + description: message.description?.trim() || undefined, + columns: message.columns.length + ? message.columns.map((column: any) => { + const name = getColumnNameByCase( + column.name, + projectByFilePath.getAdapterType(), + ); + return { + name, + description: + column.description?.trim() || undefined, + data_type: column.type?.toLowerCase(), + ...this.getTestDataByColumn( + message, + column.name, + project, + // passing column to get correct key: data_tests or tests + // https://github.com/AltimateAI/vscode-dbt-power-user/issues/1449 + { name: column.name }, + ), + ...(isQuotedIdentifier( + column.name, + projectByFilePath.getAdapterType(), + ) + ? { quote: true } + : undefined), + }; + }) + : undefined, + }; + // Models does not exist + if (existingModels?.items.length) { + parsedDocFile.addIn(["models"], newModelData); + } else { + // Models exist, but current one is new model + parsedDocFile.set("models", [newModelData]); + } + } else { + // The model already exists + this.setOrDeleteInParsedDocument( + model, + "description", + message.description?.trim(), + ); + const modelTests = this.getTestDataByModel( + message, + model.get("name") as string, + ); + this.setOrDeleteInParsedDocument( + model, + "tests", + modelTests, + ); + + message.columns.forEach((column: any) => { + const existingColumn = this.findEntityInParsedDoc( + model.get("columns") as + | YAMLSeq + | undefined, + (name: string) => isColumnNameEqual(name, column.name), + ); + + if (existingColumn) { + // ignore tests, data_tests from existing column, as it will be recreated in `getTestDataByColumn` + const { tests, data_tests, ...rest } = + existingColumn.toJSON(); + this.setOrDeleteInParsedDocument( + existingColumn, + "description", + column.description?.trim(), + ); + this.setOrDeleteInParsedDocument( + existingColumn, + "data_type", + (rest.data_type || column.type)?.toLowerCase(), + ); + const allTests = this.getTestDataByColumn( + message, + column.name, + project, + existingColumn.toJSON(), + ); + this.setOrDeleteInParsedDocument( + existingColumn, + "tests", + allTests?.tests, + ); + this.setOrDeleteInParsedDocument( + existingColumn, + "data_tests", + allTests?.data_tests, + ); + } else { const name = getColumnNameByCase( column.name, projectByFilePath.getAdapterType(), ); - return { + model.addIn(["columns"], { name, - description: column.description || undefined, + description: column.description?.trim() || undefined, data_type: column.type?.toLowerCase(), ...this.getTestDataByColumn( message, @@ -711,74 +847,16 @@ export class DocsEditViewPanel implements WebviewViewProvider { ) ? { quote: true } : undefined), - }; - }), + }); + } }); - } else { - // The model already exists - parsedDocFile.models = parsedDocFile.models.map( - (model: any) => { - if (model.name === message.name) { - model.description = message.description || undefined; - model.tests = this.getTestDataByModel( - message, - model.name, - ); - model.columns = message.columns.map((column: any) => { - const existingColumn = - model.columns && - model.columns.find((yamlColumn: any) => - isColumnNameEqual(yamlColumn.name, column.name), - ); - if (existingColumn !== undefined) { - // ignore tests, data_tests from existing column, as it will be recreated in `getTestDataByColumn` - const { tests, data_tests, ...rest } = - existingColumn; - return { - ...rest, - name: existingColumn.name, - data_type: ( - rest.data_type || column.type - )?.toLowerCase(), - description: column.description || undefined, - ...this.getTestDataByColumn( - message, - column.name, - project, - existingColumn, - ), - }; - } else { - const name = getColumnNameByCase( - column.name, - projectByFilePath.getAdapterType(), - ); - return { - name, - description: column.description || undefined, - data_type: column.type?.toLowerCase(), - ...this.getTestDataByColumn( - message, - column.name, - project, - ), - ...(isQuotedIdentifier( - column.name, - projectByFilePath.getAdapterType(), - ) - ? { quote: true } - : undefined), - }; - } - }); - } - return model; - }, - ); } // Force reload from manifest after manifest refresh this.loadedFromManifest = false; - writeFileSync(patchPath, stringify(parsedDocFile)); + writeFileSync( + patchPath, + stringify(parsedDocFile, { lineWidth: 0 }), + ); this.documentation = ( await this.docGenService.getDocumentationForCurrentActiveFile() ).documentation; From ae8f55191812d3ba0cf5bded0e67e654ccda86d7 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Thu, 24 Oct 2024 22:14:01 -0700 Subject: [PATCH 6/6] bumping version 0.47.5 -> 0.47.6 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 98cca6dcc..860cd11ab 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "bugs": { "url": "https://github.com/AltimateAI/vscode-dbt-power-user/issues" }, - "version": "0.47.5", + "version": "0.47.6", "engines": { "vscode": "^1.81.0" },