From f491c7b6e31f04bb6620b58c49b6533b547d2a88 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Mon, 18 Mar 2024 16:40:17 -0400 Subject: [PATCH 01/28] Initial commit Signed-off-by: worksofliam --- src/language/sql/formatter.ts | 26 ++++++++++++++++++++++++++ src/language/sql/tests/format.test.ts | 14 ++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 src/language/sql/formatter.ts create mode 100644 src/language/sql/tests/format.test.ts diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts new file mode 100644 index 00000000..645309c3 --- /dev/null +++ b/src/language/sql/formatter.ts @@ -0,0 +1,26 @@ +import Document from "./document"; +import { Token } from "./types"; + +export interface FormatOptions { + indent?: number; +} + +export function formatSql(document: Document, options?: FormatOptions): string { + let result: string = ``; + + for (const statement of document.statements) { + for (const token of statement.tokens) { + if (tokenIs(token, `clause`)) { + result += `\n`; + } + + result += token.value + ` `; + } + } + + return result.trimEnd(); +} + +const tokenIs = (token: Token|undefined, type: string, value?: string) => { + return (token && token.type === type && (value ? token.value?.toUpperCase() === value : true)); +} \ No newline at end of file diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts new file mode 100644 index 00000000..31ca8e14 --- /dev/null +++ b/src/language/sql/tests/format.test.ts @@ -0,0 +1,14 @@ +import { assert, expect, test } from 'vitest' +import SQLTokeniser from '../tokens' +import Document from '../document'; +import { formatSql } from '../formatter'; + +// Edit an assertion and save to see HMR in action + +test('Clause new line', () => { + const document = new Document(`select * from sample`); + + const formatted = formatSql(document); + + expect(formatted).toBe(`select * \nfrom sample`); +}); \ No newline at end of file From 6589f3937c09116e358bc1a6380fb0a5f76e5bec Mon Sep 17 00:00:00 2001 From: worksofliam Date: Mon, 13 May 2024 15:15:26 -0400 Subject: [PATCH 02/28] Use statement groups in the formatter Signed-off-by: worksofliam --- src/language/sql/formatter.ts | 15 +++++++++------ src/language/sql/tests/statements.test.ts | 1 - 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 645309c3..d9325404 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -8,13 +8,16 @@ export interface FormatOptions { export function formatSql(document: Document, options?: FormatOptions): string { let result: string = ``; - for (const statement of document.statements) { - for (const token of statement.tokens) { - if (tokenIs(token, `clause`)) { - result += `\n`; - } + const statementGroups = document.getStatementGroups(); + for (const statementGroup of statementGroups) { + for (const statement of statementGroup.statements) { + for (const token of statement.tokens) { + if (tokenIs(token, `clause`)) { + result += `\n`; + } - result += token.value + ` `; + result += token.value + ` `; + } } } diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts index a277e2ea..0b03e045 100644 --- a/src/language/sql/tests/statements.test.ts +++ b/src/language/sql/tests/statements.test.ts @@ -842,7 +842,6 @@ describe(`Object references`, () => { expect(createStatement.type).toBe(StatementType.Create); const refs = createStatement.getObjectReferences(); - console.log(refs); expect(refs.length).toBe(2); expect(refs[0].createType).toBe(`procedure`); From b1cb1b01eb972000846e86c8d2e007542fcd0872 Mon Sep 17 00:00:00 2001 From: Julia Yan Date: Thu, 4 Jul 2024 13:47:16 -0400 Subject: [PATCH 03/28] Initial formatting work --- package.json | 5 + src/language/providers/formatProvider.ts | 9 +- src/language/sql/formatter.ts | 144 +++++++++++++++++++++-- src/language/sql/tests/format.test.ts | 133 ++++++++++++++++++++- 4 files changed, 275 insertions(+), 16 deletions(-) diff --git a/package.json b/package.json index e15441f8..b25e3546 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,11 @@ "Format reserved SQL keywords in lowercase", "Format reserved SQL keywords in uppercase" ] + }, + "vscode-db2i.sqlFormat.autoIndent": { + "type": "boolean", + "description": "Automatically indent", + "default": true } } }, diff --git a/src/language/providers/formatProvider.ts b/src/language/providers/formatProvider.ts index 8741cdb3..28ed092d 100644 --- a/src/language/providers/formatProvider.ts +++ b/src/language/providers/formatProvider.ts @@ -1,13 +1,20 @@ import { Position, Range, TextEdit, languages } from "vscode"; import Statement from "../../database/statement"; +import { formatSql, IdentifierCase, KeywordCase } from "../sql/formatter"; +import Configuration from "../../configuration"; export const formatProvider = languages.registerDocumentFormattingEditProvider({language: `sql`}, { async provideDocumentFormattingEdits(document, options, token) { - const formatted = Statement.format( + const identifierCase: IdentifierCase = (Configuration.get(`sqlFormat.identifierCase`) || `preserve`); + const keywordCase: KeywordCase = (Configuration.get(`sqlFormat.keywordCase`) || `lower`); + const formatted = formatSql( document.getText(), { useTabs: !options.insertSpaces, tabWidth: options.tabSize, + identifierCase: identifierCase, + keywordCase: keywordCase, + addSemiColon: true } ); diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index d9325404..7e1fcd2a 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -1,29 +1,153 @@ +import { TextDocument } from "vscode"; import Document from "./document"; -import { Token } from "./types"; +import { StatementGroup, Token } from "./types"; +import { stat } from "fs"; +import { isToken } from "typescript"; + +export declare type KeywordCase = `preserve` | `upper` | `lower`; +export declare type IdentifierCase = `preserve` | `upper` | `lower`; export interface FormatOptions { - indent?: number; + useTabs: boolean; + tabWidth: number; // Defaults to 4 + identifierCase: IdentifierCase; + keywordCase: KeywordCase; + addSemiColon: boolean; } -export function formatSql(document: Document, options?: FormatOptions): string { +export function formatSql(textDocument: string, options?: FormatOptions, indentCounter: number = 1, isSubGroup: boolean = false): string { let result: string = ``; + let document = new Document(textDocument); + let subGroup: string = ``; + let inSubGroup: boolean = false; + let subGroupOptions: FormatOptions = {...options}; + subGroupOptions.addSemiColon = false; + let isSingleton = false; + let prevToken: Token|undefined = undefined; + let prefixSpaceAdded = true; + let numOpenBrackets = 0; + const originalIndent = indentCounter; + const statementGroups: StatementGroup[] = document.getStatementGroups(); - const statementGroups = document.getStatementGroups(); for (const statementGroup of statementGroups) { for (const statement of statementGroup.statements) { - for (const token of statement.tokens) { - if (tokenIs(token, `clause`)) { - result += `\n`; + if (statement.tokens.length == 1) { + result += statement.tokens[0].value; + isSingleton = true; + continue; + } else if (isSubGroup) { + result += `\n` + indent(options, indentCounter); + } + + for (const token of statement.tokens) { + if (tokenIs(token, `closebracket`)) { + subGroup += `)`; + if (numOpenBrackets == 1) { + inSubGroup = false; + result += formatSql(subGroup, subGroupOptions, indentCounter, true); + indentCounter--; + result += `)`; + } + numOpenBrackets--; + prevToken = token; + prefixSpaceAdded = true; + continue; + } + + if (inSubGroup) { + subGroup += ` ` + token.value; + prevToken = token; + continue; } - result += token.value + ` `; + if (tokenIs(prevToken, `closebracket`)) { + result += ` `; + } + + // If this is an ORDER BY, we need format differently + if (tokenIs(token, `clause`, `ORDER`)) { + result += `\n` + transformCase(token, options?.keywordCase) + ` `; + prefixSpaceAdded = false; + } else if (tokenIs(token, `word`, `BY`) && tokenIs(prevToken, `clause`, `ORDER`)) { + result += transformCase(token, options?.keywordCase) + `\n` + indent(options, indentCounter); + prefixSpaceAdded = true; + } else if (tokenIs(token, `clause`)) { + result += `\n` + indent(options, indentCounter - 1) + transformCase(token, options?.keywordCase) + `\n` + indent(options, indentCounter); + prefixSpaceAdded = true; + } else if (tokenIs(token, `openbracket`)) { + inSubGroup = true; + subGroup = ``; + indentCounter++; + result += ` (`; + prefixSpaceAdded = false; + numOpenBrackets++; + } else if(tokenIs(token, `comma`)) { + result += `,\n` + indent(options, indentCounter); + prefixSpaceAdded = true; + } else if (tokenIs(token, `statementType`)) { + if (isSubGroup) { + indentCounter++; + } + result += transformCase(token, options?.keywordCase) + `\n` + indent(options, indentCounter); + prefixSpaceAdded = true; + } else if(tokenIs(token, `word`)){ + if (!prefixSpaceAdded) { + result += ` `; + } + result += transformCase(token, options?.identifierCase); + prefixSpaceAdded = false; + } else if (tokenIs(token, `dot`)) { + result += token.value; + prefixSpaceAdded = true; + } else if (prefixSpaceAdded) { + result += token.value; + prefixSpaceAdded = false; + } else { + result += ` ` + token.value; + prefixSpaceAdded = false; + } + + // We need to treat ORDER BY as one clause + prevToken = token; } + if (options.addSemiColon) { + result += `;`; + } + result += `\n`; + indentCounter = originalIndent; } } - return result.trimEnd(); + if (isSubGroup && !isSingleton) { + result += indent(options, indentCounter - 1); + } else { + result = result.trimEnd(); + } + + return result; } const tokenIs = (token: Token|undefined, type: string, value?: string) => { return (token && token.type === type && (value ? token.value?.toUpperCase() === value : true)); -} \ No newline at end of file +} + +const transformCase = (token: Token|undefined, stringCase: IdentifierCase|KeywordCase|undefined) => { + if (stringCase == `upper`) { + return token.value.toUpperCase(); + } else if (stringCase == `lower`) { + return token.value.toLowerCase(); + } else { + return token.value; + } +} + +const indent = (options: FormatOptions, indentCounter: number) => { + if (indentCounter < 0) { + return ``; + } else if (options.useTabs) { + return `\t`.repeat(indentCounter); + } else { + return ` `.repeat(options.tabWidth * indentCounter); + } +} + diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts index 31ca8e14..8332689d 100644 --- a/src/language/sql/tests/format.test.ts +++ b/src/language/sql/tests/format.test.ts @@ -1,14 +1,137 @@ import { assert, expect, test } from 'vitest' import SQLTokeniser from '../tokens' import Document from '../document'; -import { formatSql } from '../formatter'; +import { FormatOptions, formatSql } from '../formatter'; + +const optionsUpper: FormatOptions = { + useTabs: false, + tabWidth: 4, + identifierCase: 'upper', + keywordCase: 'upper', + addSemiColon: true +} + +const optionsLower: FormatOptions = { + useTabs: false, + tabWidth: 4, + identifierCase: 'lower', + keywordCase: 'lower', + addSemiColon: true +} + // Edit an assertion and save to see HMR in action +test('Clause new line - upper', () => { + const sql = `select * from sample`; + const formatted = formatSql(sql, optionsUpper); + expect(formatted).toBe(`SELECT + * +FROM + SAMPLE;`); +}); + +test('Clause new line - lower', () => { + const sql = `select * from sample`; + const formatted = formatSql(sql, optionsLower); + expect(formatted).toBe(`select + * +from + sample;`); +}); + +test('Two clause statements', () => { + const sql = `select * from sample;\nselect * from sample;`; + const formatted = formatSql(sql, optionsUpper); + expect(formatted).toBe(`SELECT + * +FROM + SAMPLE; +SELECT + * +FROM + SAMPLE;`); +}); + +test('Simple multi clause', () => { + const sql = `select * from sample limit 1`; + const formatted = formatSql(sql, optionsUpper); + expect(formatted).toBe( +`SELECT + * +FROM + SAMPLE +LIMIT + 1;`); +}); + +test('Brackets', () => { + const sql = `SELECT * FROM SAMPLE(RESET_STATISTICS => 'NO', JOB_NAME_FILTER => '*ALL', DETAILED_INFO => 'NONE') WHERE UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' ORDER BY JOB_NAME_SHORT, JOB_NUMBER;`; + const formatted = formatSql(sql, optionsUpper); + expect(formatted).toBe(`SELECT + * +FROM + SAMPLE ( + RESET_STATISTICS => 'NO', + JOB_NAME_FILTER => '*ALL', + DETAILED_INFO => 'NONE' + ) +WHERE + UPPER (JOB_NAME) LIKE '%QNAVMNSRV%' +ORDER BY + JOB_NAME_SHORT, + JOB_NUMBER;`); +}); + +test('Select with columns', () => { + const sql = `SELECT ONE, TWO, THREE FROM SAMPLE2`; + const formatted = formatSql(sql, optionsUpper); + expect(formatted).toBe(`SELECT + ONE, + TWO, + THREE +FROM + SAMPLE2;`); +}); -test('Clause new line', () => { - const document = new Document(`select * from sample`); +test('Nested Select', () => { + const sql = `SELECT * FROM SAMPLE ( SELECT ONE, TWO, THREE FROM SAMPLE2 ) WHERE UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' ORDER BY JOB_NAME_SHORT, JOB_NUMBER;`; + const formatted = formatSql(sql, optionsUpper); + expect(formatted).toBe(`SELECT + * +FROM + SAMPLE ( + SELECT + ONE, + TWO, + THREE + FROM + SAMPLE2 + ) +WHERE + UPPER (JOB_NAME) LIKE '%QNAVMNSRV%' +ORDER BY + JOB_NAME_SHORT, + JOB_NUMBER;`); +}); - const formatted = formatSql(document); +test('Alter Table to Add Materialized Query (from ACS)', () => { + const sql = `ALTER TABLE table1 ADD MATERIALIZED QUERY (SELECT int_col, varchar_col FROM table3) DATA INITIALLY IMMEDIATE REFRESH DEFERRED MAINTAINED BY USER ENABLE QUERY OPTIMIZATION;`; + const formatted = formatSql(sql, optionsUpper); + expect(formatted).toBe(`ALTER + TABLE TABLE1 ADD MATERIALIZED QUERY ( + SELECT + INT_COL, + VARCHAR_COL + FROM + TABLE3 + ) DATA INITIALLY IMMEDIATE REFRESH DEFERRED MAINTAINED BY USER ENABLE QUERY OPTIMIZATION;`); +}); - expect(formatted).toBe(`select * \nfrom sample`); +test('Active jobs (from Nav)', () => { + const sql = `SELECT * FROM TABLE ( QSYS2.ACTIVE_JOB_INFO( RESET_STATISTICS => 'NO', SUBSYSTEM_LIST_FILTER => '', JOB_NAME_FILTER => '*ALL', CURRENT_USER_LIST_FILTER => '', DETAILED_INFO => 'NONE' ) ) ORDER BY SUBSYSTEM, RUN_PRIORITY, JOB_NAME_SHORT, JOB_NUMBER LIMIT 100 OFFSET 0`; + const formatted = formatSql(sql, optionsUpper); + console.log('*************'); + console.log(formatted); + console.log('*************'); + // expect(formatted).toBe(``); }); \ No newline at end of file From b65555c235f4176edd5abc7febda4d22dd8bf6e7 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 29 Aug 2024 16:03:04 -0400 Subject: [PATCH 04/28] Recursive formatter Signed-off-by: worksofliam --- src/language/sql/formatter.ts | 195 +++++++++++--------------- src/language/sql/tests/format.test.ts | 6 +- 2 files changed, 86 insertions(+), 115 deletions(-) diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 7e1fcd2a..699a36af 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -3,128 +3,112 @@ import Document from "./document"; import { StatementGroup, Token } from "./types"; import { stat } from "fs"; import { isToken } from "typescript"; +import Statement from "./statement"; +import SQLTokeniser from "./tokens"; export declare type KeywordCase = `preserve` | `upper` | `lower`; export declare type IdentifierCase = `preserve` | `upper` | `lower`; export interface FormatOptions { - useTabs: boolean; - tabWidth: number; // Defaults to 4 - identifierCase: IdentifierCase; - keywordCase: KeywordCase; - addSemiColon: boolean; + useTabs?: boolean; + tabWidth?: number; // Defaults to 4 + keywordCase?: KeywordCase; + newLineLists?: boolean; } -export function formatSql(textDocument: string, options?: FormatOptions, indentCounter: number = 1, isSubGroup: boolean = false): string { - let result: string = ``; +export function formatSql(textDocument: string, options: FormatOptions = {}): string { + let result: string[] = []; let document = new Document(textDocument); - let subGroup: string = ``; - let inSubGroup: boolean = false; - let subGroupOptions: FormatOptions = {...options}; - subGroupOptions.addSemiColon = false; - let isSingleton = false; - let prevToken: Token|undefined = undefined; - let prefixSpaceAdded = true; - let numOpenBrackets = 0; - const originalIndent = indentCounter; const statementGroups: StatementGroup[] = document.getStatementGroups(); for (const statementGroup of statementGroups) { - for (const statement of statementGroup.statements) { - if (statement.tokens.length == 1) { - result += statement.tokens[0].value; - isSingleton = true; - continue; - } else if (isSubGroup) { - result += `\n` + indent(options, indentCounter); - } - - for (const token of statement.tokens) { - if (tokenIs(token, `closebracket`)) { - subGroup += `)`; - if (numOpenBrackets == 1) { - inSubGroup = false; - result += formatSql(subGroup, subGroupOptions, indentCounter, true); - indentCounter--; - result += `)`; - } - numOpenBrackets--; - prevToken = token; - prefixSpaceAdded = true; - continue; - } + const hasBody = statementGroup.statements.length > 1; + for (let i = 0; i < statementGroup.statements.length; i++) { + const statement = statementGroup.statements[i]; + const blockStartEnd = statement.isBlockOpener() || statement.isBlockEnder(); + const withBlocks = SQLTokeniser.createBlocks(statement.tokens); + const startingIndent = hasBody ? (blockStartEnd ? 0 : 4) : 0; + + result.push(formatTokens(withBlocks, options, startingIndent) + `;`); + } + } + + return result.join(`\n`); +} + +function formatTokens(tokensWithBlocks: Token[], options: FormatOptions, baseIndent: number = 0): string { + const indent = options.tabWidth || 4; + let newLine = `\n` + ``.padEnd(baseIndent); + let res: string = newLine; - if (inSubGroup) { - subGroup += ` ` + token.value; - prevToken = token; - continue; + const updateIndent = (newIndent: number) => { + baseIndent += newIndent; + newLine = `\n` + ``.padEnd(baseIndent); + } + + for (let i = 0; i < tokensWithBlocks.length; i++) { + const cT = tokensWithBlocks[i]; + const nT = tokensWithBlocks[i + 1]; + const pT = tokensWithBlocks[i - 1]; + + switch (cT.type) { + case `block`: + if (cT.block) { + const hasClauseOrStatement = tokenIs(cT.block[0], `statementType`); + const commaCount = cT.block.filter(t => tokenIs(t, `comma`)).length; + if (cT.block.length === 1) { + res += `(${cT.block![0].value})`; + + } else if (hasClauseOrStatement) { + res += ` (`; + res += formatTokens(cT.block!, {...options, newLineLists: options.newLineLists}, baseIndent + indent); + res += `${newLine})`; + } else if (commaCount >= 2) { + res += `(`; + res += formatTokens(cT.block!, {...options, newLineLists: true}, baseIndent + indent); + res += `${newLine})`; + } else { + res += `(${formatTokens(cT.block!, options)})`; + } + } else { + throw new Error(`Block token without block`); } + break; + case `dot`: + res += cT.value; + break; + case `comma`: + res += cT.value; - if (tokenIs(prevToken, `closebracket`)) { - result += ` `; + if (options.newLineLists) { + res += newLine; } + break; - // If this is an ORDER BY, we need format differently - if (tokenIs(token, `clause`, `ORDER`)) { - result += `\n` + transformCase(token, options?.keywordCase) + ` `; - prefixSpaceAdded = false; - } else if (tokenIs(token, `word`, `BY`) && tokenIs(prevToken, `clause`, `ORDER`)) { - result += transformCase(token, options?.keywordCase) + `\n` + indent(options, indentCounter); - prefixSpaceAdded = true; - } else if (tokenIs(token, `clause`)) { - result += `\n` + indent(options, indentCounter - 1) + transformCase(token, options?.keywordCase) + `\n` + indent(options, indentCounter); - prefixSpaceAdded = true; - } else if (tokenIs(token, `openbracket`)) { - inSubGroup = true; - subGroup = ``; - indentCounter++; - result += ` (`; - prefixSpaceAdded = false; - numOpenBrackets++; - } else if(tokenIs(token, `comma`)) { - result += `,\n` + indent(options, indentCounter); - prefixSpaceAdded = true; - } else if (tokenIs(token, `statementType`)) { - if (isSubGroup) { - indentCounter++; + default: + const isKeyword = (tokenIs(cT, `statementType`) || tokenIs(cT, `clause`)); + if (isKeyword && i > 0) { + if (options.newLineLists) { + updateIndent(-indent); } - result += transformCase(token, options?.keywordCase) + `\n` + indent(options, indentCounter); - prefixSpaceAdded = true; - } else if(tokenIs(token, `word`)){ - if (!prefixSpaceAdded) { - result += ` `; - } - result += transformCase(token, options?.identifierCase); - prefixSpaceAdded = false; - } else if (tokenIs(token, `dot`)) { - result += token.value; - prefixSpaceAdded = true; - } else if (prefixSpaceAdded) { - result += token.value; - prefixSpaceAdded = false; - } else { - result += ` ` + token.value; - prefixSpaceAdded = false; + res += newLine; + } + + else if (!res.endsWith(` `) && i > 0) { + res += ` `; } - // We need to treat ORDER BY as one clause - prevToken = token; - } - if (options.addSemiColon) { - result += `;`; - } - result += `\n`; - indentCounter = originalIndent; - } - } + res += transformCase(cT, cT.type === `word` ? undefined : options.keywordCase); - if (isSubGroup && !isSingleton) { - result += indent(options, indentCounter - 1); - } else { - result = result.trimEnd(); + if (options.newLineLists && isKeyword) { + updateIndent(indent); + res += newLine; + } + break; + } } - return result; + return res; } const tokenIs = (token: Token|undefined, type: string, value?: string) => { @@ -140,14 +124,3 @@ const transformCase = (token: Token|undefined, stringCase: IdentifierCase|Keywor return token.value; } } - -const indent = (options: FormatOptions, indentCounter: number) => { - if (indentCounter < 0) { - return ``; - } else if (options.useTabs) { - return `\t`.repeat(indentCounter); - } else { - return ` `.repeat(options.tabWidth * indentCounter); - } -} - diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts index 8332689d..075d1c21 100644 --- a/src/language/sql/tests/format.test.ts +++ b/src/language/sql/tests/format.test.ts @@ -6,17 +6,15 @@ import { FormatOptions, formatSql } from '../formatter'; const optionsUpper: FormatOptions = { useTabs: false, tabWidth: 4, - identifierCase: 'upper', keywordCase: 'upper', - addSemiColon: true + newLineLists: true } const optionsLower: FormatOptions = { useTabs: false, tabWidth: 4, - identifierCase: 'lower', keywordCase: 'lower', - addSemiColon: true + newLineLists: true } From 6008ccd7d2af208738f5bbd025e8d427757ac82a Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 29 Aug 2024 16:07:38 -0400 Subject: [PATCH 05/28] Add case options back Signed-off-by: worksofliam --- src/language/providers/formatProvider.ts | 6 +++--- src/language/sql/formatter.ts | 15 ++++++++------- src/language/sql/tests/format.test.ts | 2 ++ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/language/providers/formatProvider.ts b/src/language/providers/formatProvider.ts index 28ed092d..b6424072 100644 --- a/src/language/providers/formatProvider.ts +++ b/src/language/providers/formatProvider.ts @@ -1,12 +1,12 @@ import { Position, Range, TextEdit, languages } from "vscode"; import Statement from "../../database/statement"; -import { formatSql, IdentifierCase, KeywordCase } from "../sql/formatter"; +import { formatSql, CaseOptions } from "../sql/formatter"; import Configuration from "../../configuration"; export const formatProvider = languages.registerDocumentFormattingEditProvider({language: `sql`}, { async provideDocumentFormattingEdits(document, options, token) { - const identifierCase: IdentifierCase = (Configuration.get(`sqlFormat.identifierCase`) || `preserve`); - const keywordCase: KeywordCase = (Configuration.get(`sqlFormat.keywordCase`) || `lower`); + const identifierCase: CaseOptions = (Configuration.get(`sqlFormat.identifierCase`) || `preserve`); + const keywordCase: CaseOptions = (Configuration.get(`sqlFormat.keywordCase`) || `lower`); const formatted = formatSql( document.getText(), { diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 699a36af..1f1f1404 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -7,12 +7,13 @@ import Statement from "./statement"; import SQLTokeniser from "./tokens"; -export declare type KeywordCase = `preserve` | `upper` | `lower`; -export declare type IdentifierCase = `preserve` | `upper` | `lower`; +export declare type CaseOptions = `preserve` | `upper` | `lower`; + export interface FormatOptions { useTabs?: boolean; tabWidth?: number; // Defaults to 4 - keywordCase?: KeywordCase; + keywordCase?: CaseOptions; + identifierCase?: CaseOptions; newLineLists?: boolean; } @@ -33,7 +34,7 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st } } - return result.join(`\n`); + return result.join(`\n`).trim(); } function formatTokens(tokensWithBlocks: Token[], options: FormatOptions, baseIndent: number = 0): string { @@ -94,11 +95,11 @@ function formatTokens(tokensWithBlocks: Token[], options: FormatOptions, baseInd res += newLine; } - else if (!res.endsWith(` `) && i > 0) { + else if (!res.endsWith(` `) && pT?.type !== `dot` && i > 0) { res += ` `; } - res += transformCase(cT, cT.type === `word` ? undefined : options.keywordCase); + res += transformCase(cT, cT.type === `word` ? options.identifierCase : options.keywordCase); if (options.newLineLists && isKeyword) { updateIndent(indent); @@ -115,7 +116,7 @@ const tokenIs = (token: Token|undefined, type: string, value?: string) => { return (token && token.type === type && (value ? token.value?.toUpperCase() === value : true)); } -const transformCase = (token: Token|undefined, stringCase: IdentifierCase|KeywordCase|undefined) => { +const transformCase = (token: Token|undefined, stringCase: CaseOptions|undefined) => { if (stringCase == `upper`) { return token.value.toUpperCase(); } else if (stringCase == `lower`) { diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts index 075d1c21..de29e1a6 100644 --- a/src/language/sql/tests/format.test.ts +++ b/src/language/sql/tests/format.test.ts @@ -7,6 +7,7 @@ const optionsUpper: FormatOptions = { useTabs: false, tabWidth: 4, keywordCase: 'upper', + identifierCase: 'upper', newLineLists: true } @@ -14,6 +15,7 @@ const optionsLower: FormatOptions = { useTabs: false, tabWidth: 4, keywordCase: 'lower', + identifierCase: 'lower', newLineLists: true } From f3893a987647d08c4ecd98e5be26496178d214d8 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 29 Aug 2024 16:13:44 -0400 Subject: [PATCH 06/28] Improvements for subblocks Signed-off-by: worksofliam --- src/language/sql/formatter.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 1f1f1404..22119c2d 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -2,7 +2,7 @@ import { TextDocument } from "vscode"; import Document from "./document"; import { StatementGroup, Token } from "./types"; import { stat } from "fs"; -import { isToken } from "typescript"; +import { IndexKind, isToken } from "typescript"; import Statement from "./statement"; import SQLTokeniser from "./tokens"; @@ -57,10 +57,11 @@ function formatTokens(tokensWithBlocks: Token[], options: FormatOptions, baseInd if (cT.block) { const hasClauseOrStatement = tokenIs(cT.block[0], `statementType`); const commaCount = cT.block.filter(t => tokenIs(t, `comma`)).length; + const containsSubBlock = cT.block.some(t => t.type === `block`); if (cT.block.length === 1) { res += `(${cT.block![0].value})`; - } else if (hasClauseOrStatement) { + } else if (hasClauseOrStatement || containsSubBlock) { res += ` (`; res += formatTokens(cT.block!, {...options, newLineLists: options.newLineLists}, baseIndent + indent); res += `${newLine})`; From 0a4052c6fc05abb727514591c0e1c265173476ae Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 29 Aug 2024 16:28:37 -0400 Subject: [PATCH 07/28] Fix to tokeniser to better support ORDER BY Signed-off-by: worksofliam --- src/language/sql/formatter.ts | 6 ++++-- src/language/sql/tokens.ts | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 22119c2d..ad84819d 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -34,10 +34,12 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st } } - return result.join(`\n`).trim(); + return result + .map((line) => (line[0] === `\n` ? line.substring(1) : line)) + .join(`\n`) } -function formatTokens(tokensWithBlocks: Token[], options: FormatOptions, baseIndent: number = 0): string { +function formatTokens(tokensWithBlocks: Token[], options: FormatOptions, baseIndent: number = 0): string { const indent = options.tabWidth || 4; let newLine = `\n` + ``.padEnd(baseIndent); let res: string = newLine; diff --git a/src/language/sql/tokens.ts b/src/language/sql/tokens.ts index 19837829..a0c51e1d 100644 --- a/src/language/sql/tokens.ts +++ b/src/language/sql/tokens.ts @@ -38,6 +38,14 @@ export default class SQLTokeniser { } }], becomes: `statementType`, }, + { + name: `CLAUSE-ORDER`, + match: [ + {type: `word`, match: (value: string) => {return value.toUpperCase() === `ORDER`}}, + {type: `word`, match: (value: string) => {return value.toUpperCase() === `BY`}} + ], + becomes: `clause`, + }, { name: `CLAUSE`, match: [{ type: `word`, match: (value: string) => { From 3f30c235db0951c2ed9a3bd1e132ad550d6337a0 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 29 Aug 2024 16:28:48 -0400 Subject: [PATCH 08/28] Remove unneeded spaces from test results Signed-off-by: worksofliam --- src/language/sql/tests/format.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts index de29e1a6..609a2477 100644 --- a/src/language/sql/tests/format.test.ts +++ b/src/language/sql/tests/format.test.ts @@ -70,13 +70,13 @@ test('Brackets', () => { expect(formatted).toBe(`SELECT * FROM - SAMPLE ( + SAMPLE( RESET_STATISTICS => 'NO', JOB_NAME_FILTER => '*ALL', DETAILED_INFO => 'NONE' - ) + ) WHERE - UPPER (JOB_NAME) LIKE '%QNAVMNSRV%' + UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' ORDER BY JOB_NAME_SHORT, JOB_NUMBER;`); @@ -106,9 +106,9 @@ FROM THREE FROM SAMPLE2 - ) + ) WHERE - UPPER (JOB_NAME) LIKE '%QNAVMNSRV%' + UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' ORDER BY JOB_NAME_SHORT, JOB_NUMBER;`); From f45e339ee8078652f0f9a75593455ad76daa871b Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 29 Aug 2024 16:42:20 -0400 Subject: [PATCH 09/28] Test the formatter on existing tests Signed-off-by: worksofliam --- src/language/sql/tests/statements.test.ts | 258 +++++++++++----------- 1 file changed, 132 insertions(+), 126 deletions(-) diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts index a3d7ed4a..e0c8ff36 100644 --- a/src/language/sql/tests/statements.test.ts +++ b/src/language/sql/tests/statements.test.ts @@ -2,17 +2,23 @@ import { assert, describe, expect, test } from 'vitest' import SQLTokeniser from '../tokens' import Document from '../document'; import { ClauseType, StatementType } from '../types'; +import { formatSql } from '../formatter'; -describe(`Basic statements`, () => { +const parserScenarios = describe.each([ + {newDoc: (content: string) => new Document(content)}, + {newDoc: (content: string) => new Document(formatSql(content))} +]); + +parserScenarios(`Basic statements`, ({newDoc}) => { test('One statement, no end', () => { - const document = new Document(`select * from sample`); + const document = newDoc(`select * from sample`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(4); }); test('One statement, with end', () => { - const document = new Document(`select * from sample;`); + const document = newDoc(`select * from sample;`); expect(document.statements.length).toBe(1); expect(document.statements[0].type).toBe(StatementType.Select); @@ -20,7 +26,7 @@ describe(`Basic statements`, () => { }); test('Two statements, one end', () => { - const document = new Document([ + const document = newDoc([ `select * from sample;`, `select a from b.b` ].join(`\n`)); @@ -34,7 +40,7 @@ describe(`Basic statements`, () => { }); test('Two statements, both end', () => { - const document = new Document([ + const document = newDoc([ `select * from sample;`, `select a from b.b;` ].join(`\n`)); @@ -45,7 +51,7 @@ describe(`Basic statements`, () => { }); test('Two statements, both end, with comments', () => { - const document = new Document([ + const document = newDoc([ `select * from sample; --Yep`, `select a from b.b; -- Nope` ].join(`\n`)); @@ -56,7 +62,7 @@ describe(`Basic statements`, () => { }); test('Two statements, both end, with comments, trimmed', () => { - const document = new Document([ + const document = newDoc([ ``, `select * from sample; --Yep`, ``, @@ -72,9 +78,9 @@ describe(`Basic statements`, () => { }); }); -describe(`Object references`, () => { +parserScenarios(`Object references`, ({newDoc}) => { test('SELECT: Simple unqualified object', () => { - const document = new Document(`select * from sample;`); + const document = newDoc(`select * from sample;`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(4); @@ -92,7 +98,7 @@ describe(`Object references`, () => { }); test('SELECT: Simple qualified object', () => { - const document = new Document(`select * from myschema.sample;`); + const document = newDoc(`select * from myschema.sample;`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(6); @@ -110,7 +116,7 @@ describe(`Object references`, () => { }); test('SELECT: Simple qualified object with alias', () => { - const document = new Document(`select * from myschema.sample as a;`); + const document = newDoc(`select * from myschema.sample as a;`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(8); @@ -128,7 +134,7 @@ describe(`Object references`, () => { }); test('SELECT: Simple unqualified object with alias (no AS)', () => { - const document = new Document(`select * from sample a;`); + const document = newDoc(`select * from sample a;`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(5); @@ -146,7 +152,7 @@ describe(`Object references`, () => { }); test('SELECT: Simple qualified object with alias (no AS)', () => { - const document = new Document(`select * from myschema.sample a;`); + const document = newDoc(`select * from myschema.sample a;`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(7); @@ -164,7 +170,7 @@ describe(`Object references`, () => { }); test('SELECT: Simple qualified object with alias (system naming)', () => { - const document = new Document(`select * from myschema/sample as a;`); + const document = newDoc(`select * from myschema/sample as a;`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(8); @@ -192,7 +198,7 @@ describe(`Object references`, () => { `WHERE ORCUID = CUID`, ].join(`\r\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); @@ -216,7 +222,7 @@ describe(`Object references`, () => { `WHERE LASTNAME > 'S';`, ].join(`\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); @@ -246,7 +252,7 @@ describe(`Object references`, () => { `WHERE LASTNAME > 'S'`, ].join(`\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); @@ -279,7 +285,7 @@ describe(`Object references`, () => { `WHERE LASTNAME > 'S';`, ].join(`\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); @@ -309,7 +315,7 @@ describe(`Object references`, () => { `WHERE LASTNAME > 'S'`, ].join(`\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); @@ -336,7 +342,7 @@ describe(`Object references`, () => { `SELECT * FROM A CROSS JOIN B`, ].join(`\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); @@ -366,7 +372,7 @@ describe(`Object references`, () => { `WHERE LASTNAME > 'S'`, ].join(`\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); @@ -398,7 +404,7 @@ describe(`Object references`, () => { `insert into "myschema".hashtags (tag, base_talk) values('#hi', 2);`, ].join(`\r\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(2); @@ -423,7 +429,7 @@ describe(`Object references`, () => { `delete from talks where id > 2;` ].join(`\r\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -441,7 +447,7 @@ describe(`Object references`, () => { `call create_Sql_sample('MYNEWSCHEMA');` ].join(`\r\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -459,7 +465,7 @@ describe(`Object references`, () => { `call "QSYS".create_Sql_sample('MYNEWSCHEMA');` ].join(`\r\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -480,7 +486,7 @@ describe(`Object references`, () => { ` ON DELETE SET NULL;`, ].join(`\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -509,7 +515,7 @@ describe(`Object references`, () => { ` ON DELETE SET NULL;`, ].join(`\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -539,7 +545,7 @@ describe(`Object references`, () => { ` ON DEPARTMENT (MGRNO);`, ].join(`\r\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(2); @@ -588,7 +594,7 @@ describe(`Object references`, () => { ` ON other.DEPARTMENT (MGRNO);`, ].join(`\r\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(2); @@ -635,7 +641,7 @@ describe(`Object references`, () => { ` PRIMARY KEY( COL_B ) );`, ].join(`\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -658,7 +664,7 @@ describe(`Object references`, () => { `);`, ].join(`\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -701,7 +707,7 @@ describe(`Object references`, () => { ` ;`, ].join(`\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -747,7 +753,7 @@ describe(`Object references`, () => { ` ;`, ].join(`\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -797,7 +803,7 @@ describe(`Object references`, () => { `end;`, ].join(`\n`); - const document = new Document(lines); + const document = newDoc(lines); const groups = document.getStatementGroups(); expect(groups.length).toBe(1); @@ -858,7 +864,7 @@ describe(`Object references`, () => { `end;`, ].join(`\n`); - const document = new Document(lines); + const document = newDoc(lines); const groups = document.getStatementGroups(); expect(groups.length).toBe(1); @@ -886,7 +892,7 @@ describe(`Object references`, () => { `EXTERNAL NAME LIB.PROGRAM GENERAL;`, ].join(`\n`); - const document = new Document(lines); + const document = newDoc(lines); const groups = document.getStatementGroups(); expect(groups.length).toBe(1); @@ -913,7 +919,7 @@ describe(`Object references`, () => { }); test(`DECLARE VARIABLE`, () => { - const document = new Document(`declare watsonx_response Varchar(10000) CCSID 1208;`); + const document = newDoc(`declare watsonx_response Varchar(10000) CCSID 1208;`); const groups = document.getStatementGroups(); expect(groups.length).toBe(1); @@ -928,7 +934,7 @@ describe(`Object references`, () => { }); test(`CREATE OR REPLACE VARIABLE`, () => { - const document = new Document(`create or replace variable watsonx.apiVersion varchar(10) ccsid 1208 default '2023-07-07';`); + const document = newDoc(`create or replace variable watsonx.apiVersion varchar(10) ccsid 1208 default '2023-07-07';`); const groups = document.getStatementGroups(); @@ -945,9 +951,9 @@ describe(`Object references`, () => { }); }); -describe(`Offset reference tests`, () => { +parserScenarios(`Offset reference tests`, ({newDoc}) => { test(`Writing select`, () => { - const document = new Document(`select * from sample.;`); + const document = newDoc(`select * from sample.;`); expect(document.statements.length).toBe(1); @@ -960,7 +966,7 @@ describe(`Offset reference tests`, () => { }); test(`Writing select, invalid middle`, () => { - const document = new Document(`select b. from department b;`); + const document = newDoc(`select b. from department b;`); expect(document.statements.length).toBe(1); @@ -979,7 +985,7 @@ describe(`Offset reference tests`, () => { }); }); -describe(`PL body tests`, () => { +parserScenarios(`PL body tests`, ({newDoc}) => { test(`CREATE PROCEDURE: with body`, () => { const lines = [ `CREATE PROCEDURE MEDIAN_RESULT_SET (OUT medianSalary DECIMAL(7,2))`, @@ -1011,7 +1017,7 @@ describe(`PL body tests`, () => { `END`, ].join(`\r\n`); - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; const medianResultSetProc = statements[0]; @@ -1072,7 +1078,7 @@ describe(`PL body tests`, () => { `CALL MEDIAN_RESULT_SET(12345.55);`, ].join(`\r\n`); - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; const medianResultSetProc = statements[0]; @@ -1127,7 +1133,7 @@ describe(`PL body tests`, () => { `select * from Temp02`, ].join(`\n`); - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1166,7 +1172,7 @@ describe(`PL body tests`, () => { `select * from cteme` ].join(`\r\n`); - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1193,7 +1199,7 @@ describe(`PL body tests`, () => { test(`SELECT: table function`, () => { const lines = `select * from table(qsys2.mti_info());`; - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1211,7 +1217,7 @@ describe(`PL body tests`, () => { test(`SELECT: table function with name (no AS)`, () => { const lines = `select * from table(qsys2.mti_info()) x;`; - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1229,7 +1235,7 @@ describe(`PL body tests`, () => { test(`SELECT: table function with name (with AS)`, () => { const lines = `select * from table(qsys2.mti_info()) as x;`; - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1257,7 +1263,7 @@ describe(`PL body tests`, () => { `stop;`, ].join(`\n`); - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(2); @@ -1271,9 +1277,9 @@ describe(`PL body tests`, () => { }) }); -describe(`Parameter statement tests`, () => { +parserScenarios(`Parameter statement tests`, ({newDoc}) => { test(`Single questionmark parameter test`, () => { - const document = new Document(`select * from sample where x = ?`); + const document = newDoc(`select * from sample where x = ?`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1284,7 +1290,7 @@ describe(`Parameter statement tests`, () => { }); test(`Single host parameter test`, () => { - const document = new Document(`select * from sample where x = :value`); + const document = newDoc(`select * from sample where x = :value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1295,7 +1301,7 @@ describe(`Parameter statement tests`, () => { }); test(`Single host qualified parameter test`, () => { - const document = new Document(`select * from sample where x = :struct.value`); + const document = newDoc(`select * from sample where x = :struct.value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1306,7 +1312,7 @@ describe(`Parameter statement tests`, () => { }); test(`Single INTO clause test`, () => { - const document = new Document(`select abcd into :myvar from sample where x = 1`); + const document = newDoc(`select abcd into :myvar from sample where x = 1`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1317,7 +1323,7 @@ describe(`Parameter statement tests`, () => { }); test(`Single INTO clause and host qualified parameter test`, () => { - const document = new Document(`select * into :myds from sample where x = :struct.value`); + const document = newDoc(`select * into :myds from sample where x = :struct.value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1328,7 +1334,7 @@ describe(`Parameter statement tests`, () => { }); test(`Double questionmark parameter test`, () => { - const document = new Document(`select * from sample where x = ? and y=?`); + const document = newDoc(`select * from sample where x = ? and y=?`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1339,7 +1345,7 @@ describe(`Parameter statement tests`, () => { }); test(`Double host parameter test`, () => { - const document = new Document(`select * from sample where x = :value and y=:whoop`); + const document = newDoc(`select * from sample where x = :value and y=:whoop`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1350,7 +1356,7 @@ describe(`Parameter statement tests`, () => { }); test(`Double host qualified parameter test`, () => { - const document = new Document(`select * from sample where x = :struct.value or y=:struct.val`); + const document = newDoc(`select * from sample where x = :struct.value or y=:struct.val`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1361,7 +1367,7 @@ describe(`Parameter statement tests`, () => { }); test('JSON_OBJECT parameters should not mark as embedded', () => { - const document = new Document(`values json_object('model_id': 'meta-llama/llama-2-13b-chat', 'input': 'TEXT', 'parameters': json_object('max_new_tokens': 100, 'time_limit': 1000), 'space_id': 'SPACEID')`); + const document = newDoc(`values json_object('model_id': 'meta-llama/llama-2-13b-chat', 'input': 'TEXT', 'parameters': json_object('max_new_tokens': 100, 'time_limit': 1000), 'space_id': 'SPACEID')`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1373,7 +1379,7 @@ describe(`Parameter statement tests`, () => { test(`Single questionmark parameter content test`, () => { const sql = `select * from sample where x = ?`; - const document = new Document(sql); + const document = newDoc(sql); const statements = document.statements; expect(statements.length).toBe(1); @@ -1384,7 +1390,7 @@ describe(`Parameter statement tests`, () => { }); test(`Single host parameter content test`, () => { - const document = new Document(`select * from sample where x = :value`); + const document = newDoc(`select * from sample where x = :value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1395,7 +1401,7 @@ describe(`Parameter statement tests`, () => { }); test(`Single host qualified parameter content test`, () => { - const document = new Document(`select * from sample where x = :struct.value`); + const document = newDoc(`select * from sample where x = :struct.value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1406,7 +1412,7 @@ describe(`Parameter statement tests`, () => { }); test(`Double questionmark parameter content test`, () => { - const document = new Document(`select * from sample where x = ? and y=?`); + const document = newDoc(`select * from sample where x = ? and y=?`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1417,7 +1423,7 @@ describe(`Parameter statement tests`, () => { }); test(`Double host parameter content test`, () => { - const document = new Document(`select * from sample where x = :value and y=:whoop`); + const document = newDoc(`select * from sample where x = :value and y=:whoop`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1428,7 +1434,7 @@ describe(`Parameter statement tests`, () => { }); test(`Double host qualified parameter content test`, () => { - const document = new Document(`select * from sample where x = :struct.value or y=:struct.val`); + const document = newDoc(`select * from sample where x = :struct.value or y=:struct.val`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1439,7 +1445,7 @@ describe(`Parameter statement tests`, () => { }); test(`Single INTO clause content test`, () => { - const document = new Document(`select abcd into :myvar from sample where x = 1`); + const document = newDoc(`select abcd into :myvar from sample where x = 1`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1450,7 +1456,7 @@ describe(`Parameter statement tests`, () => { }); test(`Single INTO clause and host qualified parameter content test`, () => { - const document = new Document(`select * into :myds from sample where x = :struct.value`); + const document = newDoc(`select * into :myds from sample where x = :struct.value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1461,7 +1467,7 @@ describe(`Parameter statement tests`, () => { }); test(`Double INTO clause and single host qualified parameter content test`, () => { - const document = new Document(`select x,y into :myds.x,:myds.y from sample where x = :struct.value`); + const document = newDoc(`select x,y into :myds.x,:myds.y from sample where x = :struct.value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1472,7 +1478,7 @@ describe(`Parameter statement tests`, () => { }); test(`Double INTO clause and single host qualified parameter content test`, () => { - const document = new Document(`Exec Sql select x,y into :myds.x,:myds.y from sample where x = :struct.value`); + const document = newDoc(`Exec Sql select x,y into :myds.x,:myds.y from sample where x = :struct.value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1490,7 +1496,7 @@ describe(`Parameter statement tests`, () => { ` WHERE WORKDEPT = :DEPTNO`, ].join(`\n`); - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1508,13 +1514,13 @@ describe(`Parameter statement tests`, () => { test(`Exec with basic DECLARE`, () => { const lines = [ `EXEC SQL`, - `DECLARE cursor-name SCROLL CURSOR FOR`, - `SELECT column-1, column-2`, - ` FROM table-name`, - ` WHERE column-1 = expression`, + `DECLARE cursor_name SCROLL CURSOR FOR`, + `SELECT column_1, column_2`, + `FROM table_name`, + `WHERE column_1 = expression`, ].join(`\n`); - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1523,16 +1529,16 @@ describe(`Parameter statement tests`, () => { const result = document.removeEmbeddedAreas(statement); expect(result.parameterCount).toBe(0); expect(result.content).toBe([ - `SELECT column-1, column-2`, - ` FROM table-name`, - ` WHERE column-1 = expression`, + `SELECT column_1, column_2`, + `FROM table_name`, + `WHERE column_1 = expression`, ].join(`\n`)); }); test(`Insert with INTO clause`, () => { const content = `INSERT INTO COOLSTUFF.DLRGPSNEW (DLRID, LOCATION) SELECT ID, QSYS2.ST_POINT(GPSLON, GPSLAT) FROM COOLSTUFF.DLRGPS2`; - const document = new Document(content); + const document = newDoc(content); const statements = document.statements; expect(statements.length).toBe(1); @@ -1542,9 +1548,53 @@ describe(`Parameter statement tests`, () => { expect(result.parameterCount).toBe(0); expect(result.content).toBe(content); }); + + test(`Callable blocks`, () => { + const lines = [ + `call qsys2.create_abcd();`, + `call qsys2.create_abcd(a, cool(a + b));`, + ].join(` `); + + const document = newDoc(lines); + const statements = document.statements; + + expect(statements.length).toBe(2); + + const a = statements[0]; + expect(a.type).toBe(StatementType.Call); + + const b = statements[1]; + expect(b.type).toBe(StatementType.Call); + + const blockA = a.getBlockRangeAt(23); + expect(blockA).toMatchObject({ start: 5, end: 5 }); + + const callableA = a.getCallableDetail(23); + expect(callableA).toBeDefined(); + expect(callableA.parentRef.object.schema).toBe(`qsys2`); + expect(callableA.parentRef.object.name).toBe(`create_abcd`); + + const blockB = a.getBlockRangeAt(24); + expect(blockB).toMatchObject({ start: 5, end: 5 }); + + const callableB = a.getCallableDetail(24); + expect(callableB).toBeDefined(); + expect(callableB.parentRef.object.schema).toBe(`qsys2`); + expect(callableB.parentRef.object.name).toBe(`create_abcd`); + + const blockC = b.getBlockRangeAt(49); + expect(blockC).toMatchObject({ start: 5, end: 13 }); + + const callableC = b.getCallableDetail(49, true); + expect(callableC).toBeDefined(); + expect(callableC.tokens.length).toBe(4); + expect(callableC.tokens.some(t => t.type === `block` && t.block.length === 3)).toBeTruthy(); + expect(callableC.parentRef.object.schema).toBe(`qsys2`); + expect(callableC.parentRef.object.name).toBe(`create_abcd`); + }); }); -describe(`Prefix tests`, () => { +parserScenarios(`Prefix tests`, ({newDoc}) => { test('CL prefix', () => { const content = [ `-- example`, @@ -1555,7 +1605,7 @@ describe(`Prefix tests`, () => { ` ORDER BY TOTAL_STORAGE_USED DESC FETCH FIRST 10 ROWS ONLY`, ].join(`\n`); - const document = new Document(content); + const document = newDoc(content); const statements = document.statements; expect(statements.length).toBe(1); @@ -1563,48 +1613,4 @@ describe(`Prefix tests`, () => { expect(statement.type).toBe(StatementType.Select); }); -}); - -test(`Callable blocks`, () => { - const lines = [ - `call qsys2.create_abcd();`, - `call qsys2.create_abcd(a, cool(a + b));`, - ].join(` `); - - const document = new Document(lines); - const statements = document.statements; - - expect(statements.length).toBe(2); - - const a = statements[0]; - expect(a.type).toBe(StatementType.Call); - - const b = statements[1]; - expect(b.type).toBe(StatementType.Call); - - const blockA = a.getBlockRangeAt(23); - expect(blockA).toMatchObject({ start: 5, end: 5 }); - - const callableA = a.getCallableDetail(23); - expect(callableA).toBeDefined(); - expect(callableA.parentRef.object.schema).toBe(`qsys2`); - expect(callableA.parentRef.object.name).toBe(`create_abcd`); - - const blockB = a.getBlockRangeAt(24); - expect(blockB).toMatchObject({ start: 5, end: 5 }); - - const callableB = a.getCallableDetail(24); - expect(callableB).toBeDefined(); - expect(callableB.parentRef.object.schema).toBe(`qsys2`); - expect(callableB.parentRef.object.name).toBe(`create_abcd`); - - const blockC = b.getBlockRangeAt(49); - expect(blockC).toMatchObject({ start: 5, end: 13 }); - - const callableC = b.getCallableDetail(49, true); - expect(callableC).toBeDefined(); - expect(callableC.tokens.length).toBe(4); - expect(callableC.tokens.some(t => t.type === `block` && t.block.length === 3)).toBeTruthy(); - expect(callableC.parentRef.object.schema).toBe(`qsys2`); - expect(callableC.parentRef.object.name).toBe(`create_abcd`); }); \ No newline at end of file From c241335e3aca4df335de1269f38b910c747ef494 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Fri, 30 Aug 2024 11:10:59 -0400 Subject: [PATCH 10/28] Apply formatter tests Signed-off-by: worksofliam --- src/language/sql/formatter.ts | 19 +++++++-- src/language/sql/tests/blocks.test.ts | 28 +++++++----- src/language/sql/tests/statements.test.ts | 52 +++++++++++------------ 3 files changed, 59 insertions(+), 40 deletions(-) diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index ad84819d..8905e09c 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -9,6 +9,8 @@ import SQLTokeniser from "./tokens"; export declare type CaseOptions = `preserve` | `upper` | `lower`; +const SINGLE_LINE_STATEMENT_TYPES = [`CREATE`, `DECLARE`, `SET`, `DELETE`]; + export interface FormatOptions { useTabs?: boolean; tabWidth?: number; // Defaults to 4 @@ -23,14 +25,21 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st const statementGroups: StatementGroup[] = document.getStatementGroups(); for (const statementGroup of statementGroups) { + let currentIndent = 0; const hasBody = statementGroup.statements.length > 1; for (let i = 0; i < statementGroup.statements.length; i++) { const statement = statementGroup.statements[i]; - const blockStartEnd = statement.isBlockOpener() || statement.isBlockEnder(); const withBlocks = SQLTokeniser.createBlocks(statement.tokens); - const startingIndent = hasBody ? (blockStartEnd ? 0 : 4) : 0; - result.push(formatTokens(withBlocks, options, startingIndent) + `;`); + if (statement.isBlockEnder()) { + currentIndent -= 4; + } + + result.push(formatTokens(withBlocks, options, currentIndent) + (statement.isBlockOpener() ? `` : `;`)); + + if (statement.isBlockOpener()) { + currentIndent += 4; + } } } @@ -104,7 +113,9 @@ function formatTokens(tokensWithBlocks: Token[], options: FormatOptions, baseInd res += transformCase(cT, cT.type === `word` ? options.identifierCase : options.keywordCase); - if (options.newLineLists && isKeyword) { + const isSingleLineOnly = SINGLE_LINE_STATEMENT_TYPES.some((type) => tokenIs(cT, `statementType`, type)); + + if (options.newLineLists && isKeyword && !isSingleLineOnly) { updateIndent(indent); res += newLine; } diff --git a/src/language/sql/tests/blocks.test.ts b/src/language/sql/tests/blocks.test.ts index 1da9a81d..224ec24d 100644 --- a/src/language/sql/tests/blocks.test.ts +++ b/src/language/sql/tests/blocks.test.ts @@ -1,9 +1,14 @@ -import { assert, describe, expect, test } from 'vitest' +import { describe, expect, test } from 'vitest' import Document from '../document'; -import { StatementType } from '../types'; +import { formatSql } from '../formatter'; -describe(`Block statement tests`, () => { +const parserScenarios = describe.each([ + {newDoc: (content: string) => new Document(content)}, + {newDoc: (content: string) => new Document(formatSql(content, {newLineLists: true}))} +]); + +parserScenarios(`Block statement tests`, ({newDoc}) => { test('Block start tests', () => { const lines = [ `CREATE ALIAS "TestDelimiters"."Delimited Alias" FOR "TestDelimiters"."Delimited Table";`, @@ -15,7 +20,7 @@ describe(`Block statement tests`, () => { `LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`, ].join(`\n`); - const doc = new Document(lines); + const doc = newDoc(lines); // CREATE, CREATE, RETURN, END, CREATE, SET, END expect(doc.statements.length).toBe(7); @@ -53,7 +58,7 @@ describe(`Block statement tests`, () => { `LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`, ].join(`\n`); - const doc = new Document(lines); + const doc = newDoc(lines); const t = doc.statements.length; @@ -96,7 +101,9 @@ describe(`Block statement tests`, () => { `LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`, ].join(`\r\n`); - const doc = new Document(lines); + const doc = newDoc(lines); + + console.log(doc.content) const groups = doc.getStatementGroups(); @@ -108,11 +115,12 @@ describe(`Block statement tests`, () => { const beginStatement = groups[2]; const compoundSubstring = lines.substring(beginStatement.range.start, beginStatement.range.end); + console.log({compoundSubstring, len: compoundSubstring.length}); expect(compoundSubstring).toBe(compoundStatement); }); }); -describe(`Definition tests`, () => { +parserScenarios(`Definition tests`, ({newDoc}) => { test(`Alias, function, procedure`, () => { const lines = [ `CREATE ALIAS "TestDelimiters"."Delimited Alias" FOR "TestDelimiters"."Delimited Table";`, @@ -124,7 +132,7 @@ describe(`Definition tests`, () => { `LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`, ].join(`\n`); - const doc = new Document(lines); + const doc = newDoc(lines); const defs = doc.getDefinitions(); @@ -161,7 +169,7 @@ describe(`Definition tests`, () => { `END;`, ].join(`\r\n`); - const doc = new Document(lines); + const doc = newDoc(lines); const defs = doc.getDefinitions(); @@ -245,7 +253,7 @@ describe(`Definition tests`, () => { `END ; `, ].join(`\n`); - const doc = new Document(lines); + const doc = newDoc(lines); const groups = doc.getStatementGroups(); expect(groups.length).toBe(1); diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts index e0c8ff36..cd69cdb3 100644 --- a/src/language/sql/tests/statements.test.ts +++ b/src/language/sql/tests/statements.test.ts @@ -1277,9 +1277,9 @@ parserScenarios(`PL body tests`, ({newDoc}) => { }) }); -parserScenarios(`Parameter statement tests`, ({newDoc}) => { +describe(`Parameter statement tests`, () => { test(`Single questionmark parameter test`, () => { - const document = newDoc(`select * from sample where x = ?`); + const document = new Document(`select * from sample where x = ?`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1290,7 +1290,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Single host parameter test`, () => { - const document = newDoc(`select * from sample where x = :value`); + const document = new Document(`select * from sample where x = :value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1301,7 +1301,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Single host qualified parameter test`, () => { - const document = newDoc(`select * from sample where x = :struct.value`); + const document = new Document(`select * from sample where x = :struct.value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1312,7 +1312,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Single INTO clause test`, () => { - const document = newDoc(`select abcd into :myvar from sample where x = 1`); + const document = new Document(`select abcd into :myvar from sample where x = 1`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1323,7 +1323,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Single INTO clause and host qualified parameter test`, () => { - const document = newDoc(`select * into :myds from sample where x = :struct.value`); + const document = new Document(`select * into :myds from sample where x = :struct.value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1334,7 +1334,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Double questionmark parameter test`, () => { - const document = newDoc(`select * from sample where x = ? and y=?`); + const document = new Document(`select * from sample where x = ? and y=?`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1345,7 +1345,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Double host parameter test`, () => { - const document = newDoc(`select * from sample where x = :value and y=:whoop`); + const document = new Document(`select * from sample where x = :value and y=:whoop`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1356,7 +1356,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Double host qualified parameter test`, () => { - const document = newDoc(`select * from sample where x = :struct.value or y=:struct.val`); + const document = new Document(`select * from sample where x = :struct.value or y=:struct.val`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1367,7 +1367,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test('JSON_OBJECT parameters should not mark as embedded', () => { - const document = newDoc(`values json_object('model_id': 'meta-llama/llama-2-13b-chat', 'input': 'TEXT', 'parameters': json_object('max_new_tokens': 100, 'time_limit': 1000), 'space_id': 'SPACEID')`); + const document = new Document(`values json_object('model_id': 'meta-llama/llama-2-13b-chat', 'input': 'TEXT', 'parameters': json_object('max_new_tokens': 100, 'time_limit': 1000), 'space_id': 'SPACEID')`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1379,7 +1379,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { test(`Single questionmark parameter content test`, () => { const sql = `select * from sample where x = ?`; - const document = newDoc(sql); + const document = new Document(sql); const statements = document.statements; expect(statements.length).toBe(1); @@ -1390,7 +1390,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Single host parameter content test`, () => { - const document = newDoc(`select * from sample where x = :value`); + const document = new Document(`select * from sample where x = :value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1401,7 +1401,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Single host qualified parameter content test`, () => { - const document = newDoc(`select * from sample where x = :struct.value`); + const document = new Document(`select * from sample where x = :struct.value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1412,7 +1412,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Double questionmark parameter content test`, () => { - const document = newDoc(`select * from sample where x = ? and y=?`); + const document = new Document(`select * from sample where x = ? and y=?`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1423,7 +1423,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Double host parameter content test`, () => { - const document = newDoc(`select * from sample where x = :value and y=:whoop`); + const document = new Document(`select * from sample where x = :value and y=:whoop`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1434,7 +1434,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Double host qualified parameter content test`, () => { - const document = newDoc(`select * from sample where x = :struct.value or y=:struct.val`); + const document = new Document(`select * from sample where x = :struct.value or y=:struct.val`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1445,7 +1445,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Single INTO clause content test`, () => { - const document = newDoc(`select abcd into :myvar from sample where x = 1`); + const document = new Document(`select abcd into :myvar from sample where x = 1`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1456,7 +1456,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Single INTO clause and host qualified parameter content test`, () => { - const document = newDoc(`select * into :myds from sample where x = :struct.value`); + const document = new Document(`select * into :myds from sample where x = :struct.value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1467,7 +1467,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Double INTO clause and single host qualified parameter content test`, () => { - const document = newDoc(`select x,y into :myds.x,:myds.y from sample where x = :struct.value`); + const document = new Document(`select x,y into :myds.x,:myds.y from sample where x = :struct.value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1478,7 +1478,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); test(`Double INTO clause and single host qualified parameter content test`, () => { - const document = newDoc(`Exec Sql select x,y into :myds.x,:myds.y from sample where x = :struct.value`); + const document = new Document(`Exec Sql select x,y into :myds.x,:myds.y from sample where x = :struct.value`); const statements = document.statements; expect(statements.length).toBe(1); @@ -1496,7 +1496,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { ` WHERE WORKDEPT = :DEPTNO`, ].join(`\n`); - const document = newDoc(lines); + const document = new Document(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1520,7 +1520,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { `WHERE column_1 = expression`, ].join(`\n`); - const document = newDoc(lines); + const document = new Document(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1538,7 +1538,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { test(`Insert with INTO clause`, () => { const content = `INSERT INTO COOLSTUFF.DLRGPSNEW (DLRID, LOCATION) SELECT ID, QSYS2.ST_POINT(GPSLON, GPSLAT) FROM COOLSTUFF.DLRGPS2`; - const document = newDoc(content); + const document = new Document(content); const statements = document.statements; expect(statements.length).toBe(1); @@ -1555,7 +1555,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { `call qsys2.create_abcd(a, cool(a + b));`, ].join(` `); - const document = newDoc(lines); + const document = new Document(lines); const statements = document.statements; expect(statements.length).toBe(2); @@ -1594,7 +1594,7 @@ parserScenarios(`Parameter statement tests`, ({newDoc}) => { }); }); -parserScenarios(`Prefix tests`, ({newDoc}) => { +describe(`Prefix tests`, () => { test('CL prefix', () => { const content = [ `-- example`, @@ -1605,7 +1605,7 @@ parserScenarios(`Prefix tests`, ({newDoc}) => { ` ORDER BY TOTAL_STORAGE_USED DESC FETCH FIRST 10 ROWS ONLY`, ].join(`\n`); - const document = newDoc(content); + const document = new Document(content); const statements = document.statements; expect(statements.length).toBe(1); From b0841c6f379ba13255682bab741728b959e70710 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 19 Sep 2024 16:58:59 -0400 Subject: [PATCH 11/28] Fix tests for formatter support Signed-off-by: worksofliam --- src/language/providers/formatProvider.ts | 1 - src/language/sql/formatter.ts | 11 +++++--- src/language/sql/tests/blocks.test.ts | 33 +++++++++++++++++------- src/language/sql/tests/format.test.ts | 6 ++--- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/language/providers/formatProvider.ts b/src/language/providers/formatProvider.ts index b6424072..cd6fd9a4 100644 --- a/src/language/providers/formatProvider.ts +++ b/src/language/providers/formatProvider.ts @@ -14,7 +14,6 @@ export const formatProvider = languages.registerDocumentFormattingEditProvider({ tabWidth: options.tabSize, identifierCase: identifierCase, keywordCase: keywordCase, - addSemiColon: true } ); diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 8905e09c..6efc575c 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -17,6 +17,7 @@ export interface FormatOptions { keywordCase?: CaseOptions; identifierCase?: CaseOptions; newLineLists?: boolean; + _eol?: string; } export function formatSql(textDocument: string, options: FormatOptions = {}): string { @@ -24,6 +25,8 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st let document = new Document(textDocument); const statementGroups: StatementGroup[] = document.getStatementGroups(); + options._eol = textDocument.includes(`\r\n`) ? `\r\n` : `\n`; + for (const statementGroup of statementGroups) { let currentIndent = 0; const hasBody = statementGroup.statements.length > 1; @@ -44,18 +47,18 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st } return result - .map((line) => (line[0] === `\n` ? line.substring(1) : line)) - .join(`\n`) + .map((line) => (line[0] === options._eol ? line.substring(1) : line)) + .join(options._eol) } function formatTokens(tokensWithBlocks: Token[], options: FormatOptions, baseIndent: number = 0): string { const indent = options.tabWidth || 4; - let newLine = `\n` + ``.padEnd(baseIndent); + let newLine = options._eol + ``.padEnd(baseIndent); let res: string = newLine; const updateIndent = (newIndent: number) => { baseIndent += newIndent; - newLine = `\n` + ``.padEnd(baseIndent); + newLine = options._eol + ``.padEnd(baseIndent); } for (let i = 0; i < tokensWithBlocks.length; i++) { diff --git a/src/language/sql/tests/blocks.test.ts b/src/language/sql/tests/blocks.test.ts index 224ec24d..f6f1ad70 100644 --- a/src/language/sql/tests/blocks.test.ts +++ b/src/language/sql/tests/blocks.test.ts @@ -4,11 +4,11 @@ import Document from '../document'; import { formatSql } from '../formatter'; const parserScenarios = describe.each([ - {newDoc: (content: string) => new Document(content)}, - {newDoc: (content: string) => new Document(formatSql(content, {newLineLists: true}))} + {newDoc: (content: string) => new Document(content), isFormatted: false}, + {newDoc: (content: string) => new Document(formatSql(content, {newLineLists: true})), isFormatted: true} ]); -parserScenarios(`Block statement tests`, ({newDoc}) => { +parserScenarios(`Block statement tests`, ({newDoc, isFormatted}) => { test('Block start tests', () => { const lines = [ `CREATE ALIAS "TestDelimiters"."Delimited Alias" FOR "TestDelimiters"."Delimited Table";`, @@ -103,20 +103,35 @@ parserScenarios(`Block statement tests`, ({newDoc}) => { const doc = newDoc(lines); - console.log(doc.content) - const groups = doc.getStatementGroups(); expect(groups.length).toBe(4); const aliasStatement = groups[0]; - const aliasSubstring = lines.substring(aliasStatement.range.start, aliasStatement.range.end); + const aliasSubstring = doc.content.substring(aliasStatement.range.start, aliasStatement.range.end); expect(aliasSubstring).toBe(`CREATE ALIAS "TestDelimiters"."Delimited Alias" FOR "TestDelimiters"."Delimited Table"`); + const functionStatement = groups[1]; + const functionSubstring = doc.content.substring(functionStatement.range.start, functionStatement.range.end); + + if (isFormatted) { + // TODO: + } else { + expect(functionSubstring).toBe([ + `CREATE FUNCTION "TestDelimiters"."Delimited Function" ("Delimited Parameter" INTEGER) `, + `RETURNS INTEGER LANGUAGE SQL BEGIN RETURN "Delimited Parameter"; END` + ].join(`\r\n`)) + } const beginStatement = groups[2]; - const compoundSubstring = lines.substring(beginStatement.range.start, beginStatement.range.end); - console.log({compoundSubstring, len: compoundSubstring.length}); - expect(compoundSubstring).toBe(compoundStatement); + expect(beginStatement.statements.length).toBe(9); + + if (isFormatted) { + // TODO: + + } else { + const compoundSubstring = doc.content.substring(beginStatement.range.start, beginStatement.range.end); + expect(compoundSubstring).toBe(compoundStatement); + } }); }); diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts index 609a2477..d075b906 100644 --- a/src/language/sql/tests/format.test.ts +++ b/src/language/sql/tests/format.test.ts @@ -130,8 +130,8 @@ test('Alter Table to Add Materialized Query (from ACS)', () => { test('Active jobs (from Nav)', () => { const sql = `SELECT * FROM TABLE ( QSYS2.ACTIVE_JOB_INFO( RESET_STATISTICS => 'NO', SUBSYSTEM_LIST_FILTER => '', JOB_NAME_FILTER => '*ALL', CURRENT_USER_LIST_FILTER => '', DETAILED_INFO => 'NONE' ) ) ORDER BY SUBSYSTEM, RUN_PRIORITY, JOB_NAME_SHORT, JOB_NUMBER LIMIT 100 OFFSET 0`; const formatted = formatSql(sql, optionsUpper); - console.log('*************'); - console.log(formatted); - console.log('*************'); +// console.log('*************'); +// console.log(formatted); +// console.log('*************'); // expect(formatted).toBe(``); }); \ No newline at end of file From c78cad4f9169ed899aa2051dbb1bbc8413e271c1 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 19 Sep 2024 17:46:07 -0400 Subject: [PATCH 12/28] Major cleanup of formatter Signed-off-by: worksofliam --- src/language/sql/formatter.ts | 102 ++++++++++++++-------- src/language/sql/tests/format.test.ts | 117 +++++++++++++++----------- 2 files changed, 135 insertions(+), 84 deletions(-) diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 6efc575c..05f3e39c 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -17,7 +17,6 @@ export interface FormatOptions { keywordCase?: CaseOptions; identifierCase?: CaseOptions; newLineLists?: boolean; - _eol?: string; } export function formatSql(textDocument: string, options: FormatOptions = {}): string { @@ -25,11 +24,10 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st let document = new Document(textDocument); const statementGroups: StatementGroup[] = document.getStatementGroups(); - options._eol = textDocument.includes(`\r\n`) ? `\r\n` : `\n`; + const eol = textDocument.includes(`\r\n`) ? `\r\n` : `\n`; for (const statementGroup of statementGroups) { let currentIndent = 0; - const hasBody = statementGroup.statements.length > 1; for (let i = 0; i < statementGroup.statements.length; i++) { const statement = statementGroup.statements[i]; const withBlocks = SQLTokeniser.createBlocks(statement.tokens); @@ -38,7 +36,10 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st currentIndent -= 4; } - result.push(formatTokens(withBlocks, options, currentIndent) + (statement.isBlockOpener() ? `` : `;`)); + result.push(...formatTokens(withBlocks, options).map(l => ``.padEnd(currentIndent) + l)); + if (!statement.isBlockOpener()) { + result[result.length-1] += `;` + } if (statement.isBlockOpener()) { currentIndent += 4; @@ -47,18 +48,35 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st } return result - .map((line) => (line[0] === options._eol ? line.substring(1) : line)) - .join(options._eol) + .map((line) => (line[0] === eol ? line.substring(1) : line)) + .join(eol) } -function formatTokens(tokensWithBlocks: Token[], options: FormatOptions, baseIndent: number = 0): string { +function formatTokens(tokensWithBlocks: Token[], options: FormatOptions): string[] { const indent = options.tabWidth || 4; - let newLine = options._eol + ``.padEnd(baseIndent); - let res: string = newLine; + let currentIndent = 0; + let newLines: string[] = [``]; + + const getSpacing = () => { + return ``.padEnd(currentIndent); + } + + const lastLine = () => { + return newLines[newLines.length-1]; + } + + const append = (newContent: string) => { + newLines[newLines.length-1] = newLines[newLines.length-1] + newContent; + } + + const newLine = (indentLevelChange = 0) => { + currentIndent += (indentLevelChange * indent); + newLines.push(getSpacing()); + } - const updateIndent = (newIndent: number) => { - baseIndent += newIndent; - newLine = options._eol + ``.padEnd(baseIndent); + const addSublines = (lines: string[]) => { + newLines.push(...lines.map(l => ``.padEnd(currentIndent + indent) + l)); + newLine(); } for (let i = 0; i < tokensWithBlocks.length; i++) { @@ -66,67 +84,83 @@ function formatTokens(tokensWithBlocks: Token[], options: FormatOptions, baseInd const nT = tokensWithBlocks[i + 1]; const pT = tokensWithBlocks[i - 1]; + const needsSpace = !lastLine().endsWith(` `) && pT?.type !== `dot` && i > 0; + switch (cT.type) { case `block`: if (cT.block) { const hasClauseOrStatement = tokenIs(cT.block[0], `statementType`); const commaCount = cT.block.filter(t => tokenIs(t, `comma`)).length; const containsSubBlock = cT.block.some(t => t.type === `block`); + + console.log({hasClauseOrStatement, containsSubBlock, commaCount}); + if (cT.block.length === 1) { - res += `(${cT.block![0].value})`; + append(`(${cT.block![0].value})`); } else if (hasClauseOrStatement || containsSubBlock) { - res += ` (`; - res += formatTokens(cT.block!, {...options, newLineLists: options.newLineLists}, baseIndent + indent); - res += `${newLine})`; + append(` (`); + addSublines(formatTokens(cT.block!, options)); + append(`)`); } else if (commaCount >= 2) { - res += `(`; - res += formatTokens(cT.block!, {...options, newLineLists: true}, baseIndent + indent); - res += `${newLine})`; + append(`(`) + addSublines(formatTokens(cT.block!, {...options, newLineLists: true})); + append(`)`); } else { - res += `(${formatTokens(cT.block!, options)})`; + const formattedSublines = formatTokens(cT.block!, options); + console.log({formattedSublines}); + if (formattedSublines.length === 1) { + append(`(${formattedSublines[0]})`); + } else { + append(`(`) + addSublines(formattedSublines); + append(`)`); + } } } else { throw new Error(`Block token without block`); } break; case `dot`: - res += cT.value; + append(cT.value); break; case `comma`: - res += cT.value; + append(cT.value); if (options.newLineLists) { - res += newLine; + newLine(); } break; + case `sqlName`: + if (needsSpace) { + append(` `); + } + + append(cT.value); + break; + default: const isKeyword = (tokenIs(cT, `statementType`) || tokenIs(cT, `clause`)); if (isKeyword && i > 0) { - if (options.newLineLists) { - updateIndent(-indent); - } - res += newLine; + newLine(options.newLineLists ? -1 : 0); } - else if (!res.endsWith(` `) && pT?.type !== `dot` && i > 0) { - res += ` `; + else if (needsSpace) { + append(` `); } - - res += transformCase(cT, cT.type === `word` ? options.identifierCase : options.keywordCase); + append(transformCase(cT, cT.type === `word` ? options.identifierCase : options.keywordCase)); const isSingleLineOnly = SINGLE_LINE_STATEMENT_TYPES.some((type) => tokenIs(cT, `statementType`, type)); if (options.newLineLists && isKeyword && !isSingleLineOnly) { - updateIndent(indent); - res += newLine; + newLine(1); } break; } } - return res; + return newLines; } const tokenIs = (token: Token|undefined, type: string, value?: string) => { diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts index d075b906..fa658c7e 100644 --- a/src/language/sql/tests/format.test.ts +++ b/src/language/sql/tests/format.test.ts @@ -82,56 +82,73 @@ ORDER BY JOB_NUMBER;`); }); -test('Select with columns', () => { - const sql = `SELECT ONE, TWO, THREE FROM SAMPLE2`; - const formatted = formatSql(sql, optionsUpper); - expect(formatted).toBe(`SELECT - ONE, - TWO, - THREE -FROM - SAMPLE2;`); -}); +// test('Select with columns', () => { +// const sql = `SELECT ONE, TWO, THREE FROM SAMPLE2`; +// const formatted = formatSql(sql, optionsUpper); +// expect(formatted).toBe(`SELECT +// ONE, +// TWO, +// THREE +// FROM +// SAMPLE2;`); +// }); -test('Nested Select', () => { - const sql = `SELECT * FROM SAMPLE ( SELECT ONE, TWO, THREE FROM SAMPLE2 ) WHERE UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' ORDER BY JOB_NAME_SHORT, JOB_NUMBER;`; - const formatted = formatSql(sql, optionsUpper); - expect(formatted).toBe(`SELECT - * -FROM - SAMPLE ( - SELECT - ONE, - TWO, - THREE - FROM - SAMPLE2 - ) -WHERE - UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' -ORDER BY - JOB_NAME_SHORT, - JOB_NUMBER;`); -}); +// test('Nested Select', () => { +// const sql = `SELECT * FROM SAMPLE ( SELECT ONE, TWO, THREE FROM SAMPLE2 ) WHERE UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' ORDER BY JOB_NAME_SHORT, JOB_NUMBER;`; +// const formatted = formatSql(sql, optionsUpper); +// expect(formatted).toBe(`SELECT +// * +// FROM +// SAMPLE ( +// SELECT +// ONE, +// TWO, +// THREE +// FROM +// SAMPLE2 +// ) +// WHERE +// UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' +// ORDER BY +// JOB_NAME_SHORT, +// JOB_NUMBER;`); +// }); -test('Alter Table to Add Materialized Query (from ACS)', () => { - const sql = `ALTER TABLE table1 ADD MATERIALIZED QUERY (SELECT int_col, varchar_col FROM table3) DATA INITIALLY IMMEDIATE REFRESH DEFERRED MAINTAINED BY USER ENABLE QUERY OPTIMIZATION;`; - const formatted = formatSql(sql, optionsUpper); - expect(formatted).toBe(`ALTER - TABLE TABLE1 ADD MATERIALIZED QUERY ( - SELECT - INT_COL, - VARCHAR_COL - FROM - TABLE3 - ) DATA INITIALLY IMMEDIATE REFRESH DEFERRED MAINTAINED BY USER ENABLE QUERY OPTIMIZATION;`); -}); +// test('Alter Table to Add Materialized Query (from ACS)', () => { +// const sql = `ALTER TABLE table1 ADD MATERIALIZED QUERY (SELECT int_col, varchar_col FROM table3) DATA INITIALLY IMMEDIATE REFRESH DEFERRED MAINTAINED BY USER ENABLE QUERY OPTIMIZATION;`; +// const formatted = formatSql(sql, optionsUpper); +// expect(formatted).toBe(`ALTER +// TABLE TABLE1 ADD MATERIALIZED QUERY ( +// SELECT +// INT_COL, +// VARCHAR_COL +// FROM +// TABLE3 +// ) DATA INITIALLY IMMEDIATE REFRESH DEFERRED MAINTAINED BY USER ENABLE QUERY OPTIMIZATION;`); +// }); -test('Active jobs (from Nav)', () => { - const sql = `SELECT * FROM TABLE ( QSYS2.ACTIVE_JOB_INFO( RESET_STATISTICS => 'NO', SUBSYSTEM_LIST_FILTER => '', JOB_NAME_FILTER => '*ALL', CURRENT_USER_LIST_FILTER => '', DETAILED_INFO => 'NONE' ) ) ORDER BY SUBSYSTEM, RUN_PRIORITY, JOB_NAME_SHORT, JOB_NUMBER LIMIT 100 OFFSET 0`; - const formatted = formatSql(sql, optionsUpper); -// console.log('*************'); -// console.log(formatted); -// console.log('*************'); - // expect(formatted).toBe(``); -}); \ No newline at end of file +// test(`CREATE FUNCTION: with single parameter`, () => { +// const sql = [ +// `CREATE FUNCTION "TestDelimiters"."Delimited Function" ("Delimited Parameter" INTEGER) `, +// `RETURNS INTEGER LANGUAGE SQL BEGIN RETURN "Delimited Parameter"; END;`, +// ].join(`\n`); + +// const formatted = formatSql(sql, optionsLower); + +// expect(formatted).toBe([ +// `create function "TestDelimiters"."Delimited Function"(`, +// ` "Delimited Parameter" integer`, +// `) returns integer language sql begin`, +// ` return "Delimited Parameter";`, +// `end;` +// ].join(`\n`)); +// }) + +// // test('Active jobs (from Nav)', () => { +// // const sql = `SELECT * FROM TABLE ( QSYS2.ACTIVE_JOB_INFO( RESET_STATISTICS => 'NO', SUBSYSTEM_LIST_FILTER => '', JOB_NAME_FILTER => '*ALL', CURRENT_USER_LIST_FILTER => '', DETAILED_INFO => 'NONE' ) ) ORDER BY SUBSYSTEM, RUN_PRIORITY, JOB_NAME_SHORT, JOB_NUMBER LIMIT 100 OFFSET 0`; +// // const formatted = formatSql(sql, optionsUpper); +// // // console.log('*************'); +// // // console.log(formatted); +// // // console.log('*************'); +// // // expect(formatted).toBe(``); +// // }); \ No newline at end of file From 7645c7294b9dc75a495be2b97137cb60eb40b0df Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 19 Sep 2024 17:48:32 -0400 Subject: [PATCH 13/28] Fix issue with spaces Signed-off-by: worksofliam --- src/language/sql/formatter.ts | 3 +- src/language/sql/tests/format.test.ts | 112 +++++++++++++------------- 2 files changed, 58 insertions(+), 57 deletions(-) diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 05f3e39c..a154a339 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -84,7 +84,8 @@ function formatTokens(tokensWithBlocks: Token[], options: FormatOptions): string const nT = tokensWithBlocks[i + 1]; const pT = tokensWithBlocks[i - 1]; - const needsSpace = !lastLine().endsWith(` `) && pT?.type !== `dot` && i > 0; + const currentLine = lastLine(); + const needsSpace = (currentLine.trim().length !== 0 && !currentLine.endsWith(` `)) && pT?.type !== `dot` && i > 0; switch (cT.type) { case `block`: diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts index fa658c7e..fe152f69 100644 --- a/src/language/sql/tests/format.test.ts +++ b/src/language/sql/tests/format.test.ts @@ -82,67 +82,67 @@ ORDER BY JOB_NUMBER;`); }); -// test('Select with columns', () => { -// const sql = `SELECT ONE, TWO, THREE FROM SAMPLE2`; -// const formatted = formatSql(sql, optionsUpper); -// expect(formatted).toBe(`SELECT -// ONE, -// TWO, -// THREE -// FROM -// SAMPLE2;`); -// }); +test('Select with columns', () => { + const sql = `SELECT ONE, TWO, THREE FROM SAMPLE2`; + const formatted = formatSql(sql, optionsUpper); + expect(formatted).toBe(`SELECT + ONE, + TWO, + THREE +FROM + SAMPLE2;`); +}); -// test('Nested Select', () => { -// const sql = `SELECT * FROM SAMPLE ( SELECT ONE, TWO, THREE FROM SAMPLE2 ) WHERE UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' ORDER BY JOB_NAME_SHORT, JOB_NUMBER;`; -// const formatted = formatSql(sql, optionsUpper); -// expect(formatted).toBe(`SELECT -// * -// FROM -// SAMPLE ( -// SELECT -// ONE, -// TWO, -// THREE -// FROM -// SAMPLE2 -// ) -// WHERE -// UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' -// ORDER BY -// JOB_NAME_SHORT, -// JOB_NUMBER;`); -// }); +test('Nested Select', () => { + const sql = `SELECT * FROM SAMPLE ( SELECT ONE, TWO, THREE FROM SAMPLE2 ) WHERE UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' ORDER BY JOB_NAME_SHORT, JOB_NUMBER;`; + const formatted = formatSql(sql, optionsUpper); + expect(formatted).toBe(`SELECT + * +FROM + SAMPLE ( + SELECT + ONE, + TWO, + THREE + FROM + SAMPLE2 + ) +WHERE + UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' +ORDER BY + JOB_NAME_SHORT, + JOB_NUMBER;`); +}); -// test('Alter Table to Add Materialized Query (from ACS)', () => { -// const sql = `ALTER TABLE table1 ADD MATERIALIZED QUERY (SELECT int_col, varchar_col FROM table3) DATA INITIALLY IMMEDIATE REFRESH DEFERRED MAINTAINED BY USER ENABLE QUERY OPTIMIZATION;`; -// const formatted = formatSql(sql, optionsUpper); -// expect(formatted).toBe(`ALTER -// TABLE TABLE1 ADD MATERIALIZED QUERY ( -// SELECT -// INT_COL, -// VARCHAR_COL -// FROM -// TABLE3 -// ) DATA INITIALLY IMMEDIATE REFRESH DEFERRED MAINTAINED BY USER ENABLE QUERY OPTIMIZATION;`); -// }); +test('Alter Table to Add Materialized Query (from ACS)', () => { + const sql = `ALTER TABLE table1 ADD MATERIALIZED QUERY (SELECT int_col, varchar_col FROM table3) DATA INITIALLY IMMEDIATE REFRESH DEFERRED MAINTAINED BY USER ENABLE QUERY OPTIMIZATION;`; + const formatted = formatSql(sql, optionsUpper); + expect(formatted).toBe(`ALTER + TABLE TABLE1 ADD MATERIALIZED QUERY ( + SELECT + INT_COL, + VARCHAR_COL + FROM + TABLE3 + ) DATA INITIALLY IMMEDIATE REFRESH DEFERRED MAINTAINED BY USER ENABLE QUERY OPTIMIZATION;`); +}); -// test(`CREATE FUNCTION: with single parameter`, () => { -// const sql = [ -// `CREATE FUNCTION "TestDelimiters"."Delimited Function" ("Delimited Parameter" INTEGER) `, -// `RETURNS INTEGER LANGUAGE SQL BEGIN RETURN "Delimited Parameter"; END;`, -// ].join(`\n`); +test(`CREATE FUNCTION: with single parameter`, () => { + const sql = [ + `CREATE FUNCTION "TestDelimiters"."Delimited Function" ("Delimited Parameter" INTEGER) `, + `RETURNS INTEGER LANGUAGE SQL BEGIN RETURN "Delimited Parameter"; END;`, + ].join(`\n`); -// const formatted = formatSql(sql, optionsLower); + const formatted = formatSql(sql, optionsLower); -// expect(formatted).toBe([ -// `create function "TestDelimiters"."Delimited Function"(`, -// ` "Delimited Parameter" integer`, -// `) returns integer language sql begin`, -// ` return "Delimited Parameter";`, -// `end;` -// ].join(`\n`)); -// }) + expect(formatted).toBe([ + `create function "TestDelimiters"."Delimited Function"(`, + ` "Delimited Parameter" integer`, + `) returns integer language sql begin`, + ` return "Delimited Parameter";`, + `end;` + ].join(`\n`)); +}) // // test('Active jobs (from Nav)', () => { // // const sql = `SELECT * FROM TABLE ( QSYS2.ACTIVE_JOB_INFO( RESET_STATISTICS => 'NO', SUBSYSTEM_LIST_FILTER => '', JOB_NAME_FILTER => '*ALL', CURRENT_USER_LIST_FILTER => '', DETAILED_INFO => 'NONE' ) ) ORDER BY SUBSYSTEM, RUN_PRIORITY, JOB_NAME_SHORT, JOB_NUMBER LIMIT 100 OFFSET 0`; From 03a9b76ccaf3174d82583cbaed9262dc1c4e40e1 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 19 Sep 2024 21:13:33 -0400 Subject: [PATCH 14/28] Refactor isBlock to isCompound Signed-off-by: worksofliam --- src/language/sql/document.ts | 4 ++-- src/language/sql/formatter.ts | 6 +++--- src/language/sql/statement.ts | 4 ++-- src/language/sql/tests/blocks.test.ts | 14 +++++++------- src/language/sql/tests/statements.test.ts | 6 +++--- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/language/sql/document.ts b/src/language/sql/document.ts index 2bbf41df..34e1c533 100644 --- a/src/language/sql/document.ts +++ b/src/language/sql/document.ts @@ -102,7 +102,7 @@ export default class Document { let depth = 0; for (const statement of this.statements) { - if (statement.isBlockEnder()) { + if (statement.isCompoundEnd()) { if (depth > 0) { currentGroup.push(statement); @@ -118,7 +118,7 @@ export default class Document { currentGroup = []; } } else - if (statement.isBlockOpener()) { + if (statement.isCompoundStart()) { if (depth > 0) { currentGroup.push(statement); } else { diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index a154a339..9de18635 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -32,16 +32,16 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st const statement = statementGroup.statements[i]; const withBlocks = SQLTokeniser.createBlocks(statement.tokens); - if (statement.isBlockEnder()) { + if (statement.isCompoundEnd()) { currentIndent -= 4; } result.push(...formatTokens(withBlocks, options).map(l => ``.padEnd(currentIndent) + l)); - if (!statement.isBlockOpener()) { + if (!statement.isCompoundStart()) { result[result.length-1] += `;` } - if (statement.isBlockOpener()) { + if (statement.isCompoundStart()) { currentIndent += 4; } } diff --git a/src/language/sql/statement.ts b/src/language/sql/statement.ts index e40a84a1..a72d782c 100644 --- a/src/language/sql/statement.ts +++ b/src/language/sql/statement.ts @@ -35,7 +35,7 @@ export default class Statement { } } - isBlockOpener() { + isCompoundStart() { if (this.tokens.length === 1 && tokenIs(this.tokens[0], `keyword`, `BEGIN`)) { return true; } @@ -51,7 +51,7 @@ export default class Statement { return false; } - isBlockEnder() { + isCompoundEnd() { return this.type === StatementType.End && this.tokens.length === 1; } diff --git a/src/language/sql/tests/blocks.test.ts b/src/language/sql/tests/blocks.test.ts index f6f1ad70..b25064dd 100644 --- a/src/language/sql/tests/blocks.test.ts +++ b/src/language/sql/tests/blocks.test.ts @@ -26,13 +26,13 @@ parserScenarios(`Block statement tests`, ({newDoc, isFormatted}) => { expect(doc.statements.length).toBe(7); const aliasDef = doc.statements[0]; - expect(aliasDef.isBlockOpener()).toBeFalsy(); + expect(aliasDef.isCompoundStart()).toBeFalsy(); const functionDef = doc.statements[1]; - expect(functionDef.isBlockOpener()).toBeTruthy(); + expect(functionDef.isCompoundStart()).toBeTruthy(); const procedureDef = doc.statements[4]; - expect(procedureDef.isBlockOpener()).toBeTruthy(); + expect(procedureDef.isCompoundStart()).toBeTruthy(); }); test('Compound statement test', () => { @@ -63,16 +63,16 @@ parserScenarios(`Block statement tests`, ({newDoc, isFormatted}) => { const t = doc.statements.length; const aliasDef = doc.statements[0]; - expect(aliasDef.isBlockOpener()).toBeFalsy(); + expect(aliasDef.isCompoundStart()).toBeFalsy(); const functionDef = doc.statements[1]; - expect(functionDef.isBlockOpener()).toBeTruthy(); + expect(functionDef.isCompoundStart()).toBeTruthy(); const functionEnd = doc.statements[3]; - expect(functionEnd.isBlockEnder()).toBeTruthy(); + expect(functionEnd.isCompoundEnd()).toBeTruthy(); const beginBlock = doc.statements[4]; - expect(beginBlock.isBlockOpener()).toBeTruthy(); + expect(beginBlock.isCompoundStart()).toBeTruthy(); }); test('Statement groups', () => { diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts index 85c3f4f9..862aa233 100644 --- a/src/language/sql/tests/statements.test.ts +++ b/src/language/sql/tests/statements.test.ts @@ -1163,7 +1163,7 @@ parserScenarios(`PL body tests`, ({newDoc}) => { const medianResultSetProc = statements[0]; expect(medianResultSetProc.type).toBe(StatementType.Create); - expect(medianResultSetProc.isBlockOpener()).toBe(true); + expect(medianResultSetProc.isCompoundStart()).toBe(true); const parms = medianResultSetProc.getRoutineParameters(); expect(parms.length).toBe(1); @@ -1224,7 +1224,7 @@ parserScenarios(`PL body tests`, ({newDoc}) => { const medianResultSetProc = statements[0]; expect(medianResultSetProc.type).toBe(StatementType.Create); - expect(medianResultSetProc.isBlockOpener()).toBe(true); + expect(medianResultSetProc.isCompoundStart()).toBe(true); const parms = medianResultSetProc.getRoutineParameters(); expect(parms.length).toBe(1); @@ -1243,7 +1243,7 @@ parserScenarios(`PL body tests`, ({newDoc}) => { const callStatement = statements[statements.length - 1]; expect(callStatement.type).toBe(StatementType.Call); - expect(callStatement.isBlockOpener()).toBe(false); + expect(callStatement.isCompoundStart()).toBe(false); const blockParent = callStatement.getCallableDetail(callStatement.tokens[3].range.start); expect(blockParent).toBeDefined(); From dc371e8e177bdb53e069c53182f3be4c5810d2bc Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 19 Sep 2024 21:19:48 -0400 Subject: [PATCH 15/28] Start of support for statement label Signed-off-by: worksofliam --- src/language/sql/statement.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/language/sql/statement.ts b/src/language/sql/statement.ts index a72d782c..018b271c 100644 --- a/src/language/sql/statement.ts +++ b/src/language/sql/statement.ts @@ -7,6 +7,7 @@ const tokenIs = (token: Token|undefined, type: string, value?: string) => { export default class Statement { public type: StatementType = StatementType.Unknown; + private label: string|undefined; constructor(public tokens: Token[], public range: IRange) { this.tokens = this.tokens.filter(newToken => newToken.type !== `newline`); @@ -19,6 +20,12 @@ export default class Statement { first = this.tokens[2]; } + if (tokenIs(first, `word`) && tokenIs(this.tokens[1], `colon`)) { + // Possible label? + this.label = first.value; + first = this.tokens[2]; + } + if (tokenIs(first, `statementType`) || tokenIs(first, `keyword`, `END`) || tokenIs(first, `keyword`, `BEGIN`)) { const wordValue = first.value?.toUpperCase(); From 60591fd0bbdde1af2ba75c6390744870bb929a96 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 19 Sep 2024 21:40:02 -0400 Subject: [PATCH 16/28] Support for more statement types Signed-off-by: worksofliam --- src/language/sql/formatter.ts | 3 -- src/language/sql/statement.ts | 6 ++-- src/language/sql/tests/statements.test.ts | 12 +++++-- src/language/sql/types.ts | 40 +++++++++++++++++++++-- 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 9de18635..1962233f 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -94,8 +94,6 @@ function formatTokens(tokensWithBlocks: Token[], options: FormatOptions): string const commaCount = cT.block.filter(t => tokenIs(t, `comma`)).length; const containsSubBlock = cT.block.some(t => t.type === `block`); - console.log({hasClauseOrStatement, containsSubBlock, commaCount}); - if (cT.block.length === 1) { append(`(${cT.block![0].value})`); @@ -109,7 +107,6 @@ function formatTokens(tokensWithBlocks: Token[], options: FormatOptions): string append(`)`); } else { const formattedSublines = formatTokens(cT.block!, options); - console.log({formattedSublines}); if (formattedSublines.length === 1) { append(`(${formattedSublines[0]})`); } else { diff --git a/src/language/sql/statement.ts b/src/language/sql/statement.ts index 018b271c..5d5dbbe9 100644 --- a/src/language/sql/statement.ts +++ b/src/language/sql/statement.ts @@ -26,11 +26,9 @@ export default class Statement { first = this.tokens[2]; } - if (tokenIs(first, `statementType`) || tokenIs(first, `keyword`, `END`) || tokenIs(first, `keyword`, `BEGIN`)) { - const wordValue = first.value?.toUpperCase(); + const wordValue = first.value?.toUpperCase(); - this.type = StatementTypeWord[wordValue]; - } + this.type = StatementTypeWord[wordValue] || StatementType.Unknown; switch (this.type) { case StatementType.Create: diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts index 862aa233..023195df 100644 --- a/src/language/sql/tests/statements.test.ts +++ b/src/language/sql/tests/statements.test.ts @@ -1065,10 +1065,10 @@ parserScenarios(`Object references`, ({newDoc}) => { expect(group.statements.map(s => s.type)).toEqual([ 'Create', 'Declare', 'Declare', 'Call', + 'Loop', 'Set', 'Unknown', 'Unknown', - 'Unknown', 'Unknown', - 'Unknown', 'Call', - 'Unknown', 'End', + 'If', 'Call', + 'Leave', 'End', 'Call', 'End', 'Unknown', 'End' ]); @@ -1184,6 +1184,12 @@ parserScenarios(`PL body tests`, ({newDoc}) => { // END expect(endStatements[1].tokens.length).toBe(1); + + const whileStatement = statements.find(stmt => stmt.type === StatementType.While); + expect(whileStatement).toBeDefined(); + + const fetchStatement = statements.find(stmt => stmt.type === StatementType.Fetch); + expect(fetchStatement).toBeDefined(); }); test(`CREATE PROCEDURE followed by CALL statement`, () => { diff --git a/src/language/sql/types.ts b/src/language/sql/types.ts index b79f1479..e7a077b9 100644 --- a/src/language/sql/types.ts +++ b/src/language/sql/types.ts @@ -3,6 +3,7 @@ import Statement from "./statement"; export enum StatementType { Unknown = "Unknown", Create = "Create", + Close = "Close", Insert = "Insert", Select = "Select", With = "With", @@ -13,7 +14,25 @@ export enum StatementType { Drop = "Drop", End = "End", Call = "Call", - Alter = "Alter" + Alter = "Alter", + Case = "Case", + Fetch = "Fetch", + For = "For", + Get = "Get", + Goto = "Goto", + If = "If", + Include = "Include", + Iterate = "Iterate", + Leave = "Leave", + Loop = "Loop", + Open = "Open", + Pipe = "Pipe", + Repeat = "Repeat", + Resignal = "Resignal", + Return = "Return", + Signal = "Signal", + Set = "Set", + While = "While" } export const StatementTypeWord = { @@ -28,7 +47,24 @@ export const StatementTypeWord = { 'END': StatementType.End, 'CALL': StatementType.Call, 'BEGIN': StatementType.Begin, - 'ALTER': StatementType.Alter + 'ALTER': StatementType.Alter, + 'CASE': StatementType.Case, + 'FOR': StatementType.For, + 'FETCH': StatementType.Fetch, + 'GET': StatementType.Get, + 'GOTO': StatementType.Goto, + 'IF': StatementType.If, + 'INCLUDE': StatementType.Include, + 'ITERATE': StatementType.Iterate, + 'LEAVE': StatementType.Leave, + 'LOOP': StatementType.Loop, + 'PIPE': StatementType.Pipe, + 'REPEAT': StatementType.Repeat, + 'RESIGNAL': StatementType.Resignal, + 'RETURN': StatementType.Return, + 'SIGNAL': StatementType.Signal, + 'SET': StatementType.Set, + 'WHILE': StatementType.While, }; export enum ClauseType { From 77420464216e40d65028b588903efec41d4ac936 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 19 Sep 2024 21:45:21 -0400 Subject: [PATCH 17/28] Start of condition support Signed-off-by: worksofliam --- src/language/sql/statement.ts | 8 ++++++++ src/language/sql/types.ts | 8 +++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/language/sql/statement.ts b/src/language/sql/statement.ts index 5d5dbbe9..d06a00f1 100644 --- a/src/language/sql/statement.ts +++ b/src/language/sql/statement.ts @@ -56,6 +56,14 @@ export default class Statement { return false; } + isConditionStart() { + return [StatementType.If, StatementType.While, StatementType.Loop, StatementType.For, StatementType.Case, StatementType.When].includes(this.type); + } + + isConditionEnd() { + return this.type === StatementType.End && this.tokens.length > 1; + } + isCompoundEnd() { return this.type === StatementType.End && this.tokens.length === 1; } diff --git a/src/language/sql/types.ts b/src/language/sql/types.ts index e7a077b9..c81b7349 100644 --- a/src/language/sql/types.ts +++ b/src/language/sql/types.ts @@ -13,6 +13,8 @@ export enum StatementType { Begin = "Begin", Drop = "Drop", End = "End", + Else = "Else", + Elseif = "Elseif", Call = "Call", Alter = "Alter", Case = "Case", @@ -32,7 +34,8 @@ export enum StatementType { Return = "Return", Signal = "Signal", Set = "Set", - While = "While" + While = "While", + When = "When" } export const StatementTypeWord = { @@ -45,6 +48,8 @@ export const StatementTypeWord = { 'DECLARE': StatementType.Declare, 'DROP': StatementType.Drop, 'END': StatementType.End, + 'ELSE': StatementType.Else, + 'ELSEIF': StatementType.Elseif, 'CALL': StatementType.Call, 'BEGIN': StatementType.Begin, 'ALTER': StatementType.Alter, @@ -65,6 +70,7 @@ export const StatementTypeWord = { 'SIGNAL': StatementType.Signal, 'SET': StatementType.Set, 'WHILE': StatementType.While, + 'WHEN': StatementType.When, }; export enum ClauseType { From 4b6b4e628f3e557f8eeeb1ac738b7d73bb35a77c Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 19 Sep 2024 21:50:55 -0400 Subject: [PATCH 18/28] Fix bug with parameters Signed-off-by: worksofliam --- src/language/sql/formatter.ts | 16 ++++++++++++++-- src/language/sql/tests/format.test.ts | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 1962233f..28ab61c1 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -1,6 +1,6 @@ import { TextDocument } from "vscode"; import Document from "./document"; -import { StatementGroup, Token } from "./types"; +import { StatementGroup, StatementType, StatementTypeWord, Token } from "./types"; import { stat } from "fs"; import { IndexKind, isToken } from "typescript"; import Statement from "./statement"; @@ -53,9 +53,21 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st } function formatTokens(tokensWithBlocks: Token[], options: FormatOptions): string[] { + let possibleType = StatementType.Unknown; const indent = options.tabWidth || 4; let currentIndent = 0; let newLines: string[] = [``]; + let typeToken: Token; + + if (tokensWithBlocks.length > 2 && tokensWithBlocks[1].type === `colon`) { + typeToken = tokensWithBlocks[2]; + } else { + typeToken = tokensWithBlocks[0]; + } + + if (typeToken && typeToken.value) { + possibleType = StatementTypeWord[typeToken.value.toUpperCase()] || StatementType.Unknown; + } const getSpacing = () => { return ``.padEnd(currentIndent); @@ -107,7 +119,7 @@ function formatTokens(tokensWithBlocks: Token[], options: FormatOptions): string append(`)`); } else { const formattedSublines = formatTokens(cT.block!, options); - if (formattedSublines.length === 1) { + if (formattedSublines.length === 1 && possibleType !== StatementType.Create) { append(`(${formattedSublines[0]})`); } else { append(`(`) diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts index fe152f69..9b5482b7 100644 --- a/src/language/sql/tests/format.test.ts +++ b/src/language/sql/tests/format.test.ts @@ -137,7 +137,7 @@ test(`CREATE FUNCTION: with single parameter`, () => { expect(formatted).toBe([ `create function "TestDelimiters"."Delimited Function"(`, - ` "Delimited Parameter" integer`, + ` "Delimited Parameter" integer`, `) returns integer language sql begin`, ` return "Delimited Parameter";`, `end;` From fd12b6a821092f78f0dec505c9c9d1bb2f36a60a Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 19 Sep 2024 21:54:31 -0400 Subject: [PATCH 19/28] Refactor formatProvider and formatter to use indentWidth instead of tabWidth Signed-off-by: worksofliam --- src/language/providers/formatProvider.ts | 3 +-- src/language/sql/formatter.ts | 10 ++-------- src/language/sql/tests/format.test.ts | 21 +++++++++------------ 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/src/language/providers/formatProvider.ts b/src/language/providers/formatProvider.ts index cd6fd9a4..f86efd75 100644 --- a/src/language/providers/formatProvider.ts +++ b/src/language/providers/formatProvider.ts @@ -10,8 +10,7 @@ export const formatProvider = languages.registerDocumentFormattingEditProvider({ const formatted = formatSql( document.getText(), { - useTabs: !options.insertSpaces, - tabWidth: options.tabSize, + indentWidth: options.tabSize, identifierCase: identifierCase, keywordCase: keywordCase, } diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 28ab61c1..fcaee95a 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -1,19 +1,13 @@ -import { TextDocument } from "vscode"; import Document from "./document"; import { StatementGroup, StatementType, StatementTypeWord, Token } from "./types"; -import { stat } from "fs"; -import { IndexKind, isToken } from "typescript"; -import Statement from "./statement"; import SQLTokeniser from "./tokens"; - export declare type CaseOptions = `preserve` | `upper` | `lower`; const SINGLE_LINE_STATEMENT_TYPES = [`CREATE`, `DECLARE`, `SET`, `DELETE`]; export interface FormatOptions { - useTabs?: boolean; - tabWidth?: number; // Defaults to 4 + indentWidth?: number; // Defaults to 4 keywordCase?: CaseOptions; identifierCase?: CaseOptions; newLineLists?: boolean; @@ -54,7 +48,7 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st function formatTokens(tokensWithBlocks: Token[], options: FormatOptions): string[] { let possibleType = StatementType.Unknown; - const indent = options.tabWidth || 4; + const indent = options.indentWidth || 4; let currentIndent = 0; let newLines: string[] = [``]; let typeToken: Token; diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts index 9b5482b7..0107fc96 100644 --- a/src/language/sql/tests/format.test.ts +++ b/src/language/sql/tests/format.test.ts @@ -4,16 +4,14 @@ import Document from '../document'; import { FormatOptions, formatSql } from '../formatter'; const optionsUpper: FormatOptions = { - useTabs: false, - tabWidth: 4, + indentWidth: 4, keywordCase: 'upper', identifierCase: 'upper', newLineLists: true } const optionsLower: FormatOptions = { - useTabs: false, - tabWidth: 4, + indentWidth: 4, keywordCase: 'lower', identifierCase: 'lower', newLineLists: true @@ -144,11 +142,10 @@ test(`CREATE FUNCTION: with single parameter`, () => { ].join(`\n`)); }) -// // test('Active jobs (from Nav)', () => { -// // const sql = `SELECT * FROM TABLE ( QSYS2.ACTIVE_JOB_INFO( RESET_STATISTICS => 'NO', SUBSYSTEM_LIST_FILTER => '', JOB_NAME_FILTER => '*ALL', CURRENT_USER_LIST_FILTER => '', DETAILED_INFO => 'NONE' ) ) ORDER BY SUBSYSTEM, RUN_PRIORITY, JOB_NAME_SHORT, JOB_NUMBER LIMIT 100 OFFSET 0`; -// // const formatted = formatSql(sql, optionsUpper); -// // // console.log('*************'); -// // // console.log(formatted); -// // // console.log('*************'); -// // // expect(formatted).toBe(``); -// // }); \ No newline at end of file +test('Active jobs (from Nav)', () => { + const sql = `SELECT * FROM TABLE ( QSYS2.ACTIVE_JOB_INFO( RESET_STATISTICS => 'NO', SUBSYSTEM_LIST_FILTER => '', JOB_NAME_FILTER => '*ALL', CURRENT_USER_LIST_FILTER => '', DETAILED_INFO => 'NONE' ) ) ORDER BY SUBSYSTEM, RUN_PRIORITY, JOB_NAME_SHORT, JOB_NUMBER LIMIT 100 OFFSET 0`; + const formatted = formatSql(sql, optionsUpper); + console.log('*************'); + console.log(formatted); + console.log('*************'); +}); \ No newline at end of file From 801a6a143d8de0bfaf492ea13bb8741bbb1d9654 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Fri, 20 Sep 2024 08:04:58 -0400 Subject: [PATCH 20/28] Formatter support for conditional statements Signed-off-by: worksofliam --- src/language/sql/formatter.ts | 20 +++++++++++--------- src/language/sql/tests/blocks.test.ts | 25 +++++++++++++++++++++---- src/language/sql/tests/format.test.ts | 6 +++--- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index fcaee95a..d42ea2be 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -4,8 +4,6 @@ import SQLTokeniser from "./tokens"; export declare type CaseOptions = `preserve` | `upper` | `lower`; -const SINGLE_LINE_STATEMENT_TYPES = [`CREATE`, `DECLARE`, `SET`, `DELETE`]; - export interface FormatOptions { indentWidth?: number; // Defaults to 4 keywordCase?: CaseOptions; @@ -13,6 +11,8 @@ export interface FormatOptions { newLineLists?: boolean; } +const SINGLE_LINE_STATEMENT_TYPES: StatementType[] = [StatementType.Create, StatementType.Declare, StatementType.Set, StatementType.Delete]; + export function formatSql(textDocument: string, options: FormatOptions = {}): string { let result: string[] = []; let document = new Document(textDocument); @@ -26,7 +26,7 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st const statement = statementGroup.statements[i]; const withBlocks = SQLTokeniser.createBlocks(statement.tokens); - if (statement.isCompoundEnd()) { + if (statement.isCompoundEnd() || statement.isConditionEnd()) { currentIndent -= 4; } @@ -35,7 +35,7 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st result[result.length-1] += `;` } - if (statement.isCompoundStart()) { + if (statement.isCompoundStart() || statement.isConditionStart()) { currentIndent += 4; } } @@ -63,6 +63,9 @@ function formatTokens(tokensWithBlocks: Token[], options: FormatOptions): string possibleType = StatementTypeWord[typeToken.value.toUpperCase()] || StatementType.Unknown; } + + const isSingleLineOnly = SINGLE_LINE_STATEMENT_TYPES.includes(possibleType); + const getSpacing = () => { return ``.padEnd(currentIndent); } @@ -145,19 +148,18 @@ function formatTokens(tokensWithBlocks: Token[], options: FormatOptions): string break; default: - const isKeyword = (tokenIs(cT, `statementType`) || tokenIs(cT, `clause`)); - if (isKeyword && i > 0) { + const isKeyword = ((tokenIs(cT, `statementType`) || tokenIs(cT, `clause`))); + if (isKeyword && i > 0 && isSingleLineOnly === false) { newLine(options.newLineLists ? -1 : 0); } else if (needsSpace) { append(` `); } + append(transformCase(cT, cT.type === `word` ? options.identifierCase : options.keywordCase)); - const isSingleLineOnly = SINGLE_LINE_STATEMENT_TYPES.some((type) => tokenIs(cT, `statementType`, type)); - - if (options.newLineLists && isKeyword && !isSingleLineOnly) { + if (options.newLineLists && isKeyword && isSingleLineOnly === false) { newLine(1); } break; diff --git a/src/language/sql/tests/blocks.test.ts b/src/language/sql/tests/blocks.test.ts index b25064dd..43c6ed76 100644 --- a/src/language/sql/tests/blocks.test.ts +++ b/src/language/sql/tests/blocks.test.ts @@ -115,7 +115,13 @@ parserScenarios(`Block statement tests`, ({newDoc, isFormatted}) => { const functionSubstring = doc.content.substring(functionStatement.range.start, functionStatement.range.end); if (isFormatted) { - // TODO: + expect(functionSubstring).toBe([ + `CREATE FUNCTION "TestDelimiters"."Delimited Function"(`, + ` "Delimited Parameter" INTEGER`, + `) RETURNS INTEGER LANGUAGE SQL BEGIN`, + ` RETURN "Delimited Parameter";`, + `END`, + ].join(`\r\n`)); } else { expect(functionSubstring).toBe([ `CREATE FUNCTION "TestDelimiters"."Delimited Function" ("Delimited Parameter" INTEGER) `, @@ -124,12 +130,23 @@ parserScenarios(`Block statement tests`, ({newDoc, isFormatted}) => { } const beginStatement = groups[2]; expect(beginStatement.statements.length).toBe(9); + const compoundSubstring = doc.content.substring(beginStatement.range.start, beginStatement.range.end); if (isFormatted) { - // TODO: - + expect(compoundSubstring).toBe([ + `BEGIN`, + ` DECLARE already_exists SMALLINT DEFAULT 0;`, + ` DECLARE dup_object_hdlr CONDITION FOR SQLSTATE '42710';`, + ` DECLARE CONTINUE HANDLER FOR dup_object_hdlr SET already_exists = 1;`, + ` CREATE TABLE table1(`, + ` col1 INT`, + ` );`, + ` IF already_exists > 0 THEN;`, + ` DELETE FROM table1;`, + ` END IF;`, + `END`, + ].join(`\r\n`)); } else { - const compoundSubstring = doc.content.substring(beginStatement.range.start, beginStatement.range.end); expect(compoundSubstring).toBe(compoundStatement); } }); diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts index 0107fc96..7e1ef625 100644 --- a/src/language/sql/tests/format.test.ts +++ b/src/language/sql/tests/format.test.ts @@ -145,7 +145,7 @@ test(`CREATE FUNCTION: with single parameter`, () => { test('Active jobs (from Nav)', () => { const sql = `SELECT * FROM TABLE ( QSYS2.ACTIVE_JOB_INFO( RESET_STATISTICS => 'NO', SUBSYSTEM_LIST_FILTER => '', JOB_NAME_FILTER => '*ALL', CURRENT_USER_LIST_FILTER => '', DETAILED_INFO => 'NONE' ) ) ORDER BY SUBSYSTEM, RUN_PRIORITY, JOB_NAME_SHORT, JOB_NUMBER LIMIT 100 OFFSET 0`; const formatted = formatSql(sql, optionsUpper); - console.log('*************'); - console.log(formatted); - console.log('*************'); + // console.log('*************'); + // console.log(formatted); + // console.log('*************'); }); \ No newline at end of file From 4b89edcf455ae38999590aac86c75e29f76d4bf7 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Fri, 20 Sep 2024 08:26:43 -0400 Subject: [PATCH 21/28] Support for improved spacing Signed-off-by: worksofliam --- package.json | 7 +-- src/language/providers/formatProvider.ts | 1 + src/language/sql/formatter.ts | 12 +++++- src/language/sql/tests/format.test.ts | 55 ++++++++++++++++++++++-- src/language/sql/tokens.ts | 5 +++ 5 files changed, 70 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 9173c1a7..17f9fa2a 100644 --- a/package.json +++ b/package.json @@ -125,11 +125,6 @@ "Format reserved SQL keywords in lowercase", "Format reserved SQL keywords in uppercase" ] - }, - "vscode-db2i.sqlFormat.autoIndent": { - "type": "boolean", - "description": "Automatically indent", - "default": true } } }, @@ -1252,4 +1247,4 @@ } ] } -} +} \ No newline at end of file diff --git a/src/language/providers/formatProvider.ts b/src/language/providers/formatProvider.ts index f86efd75..5eb03eef 100644 --- a/src/language/providers/formatProvider.ts +++ b/src/language/providers/formatProvider.ts @@ -13,6 +13,7 @@ export const formatProvider = languages.registerDocumentFormattingEditProvider({ indentWidth: options.tabSize, identifierCase: identifierCase, keywordCase: keywordCase, + spaceBetweenStatements: true } ); diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index d42ea2be..00c6e4b8 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -9,9 +9,10 @@ export interface FormatOptions { keywordCase?: CaseOptions; identifierCase?: CaseOptions; newLineLists?: boolean; + spaceBetweenStatements?: boolean } -const SINGLE_LINE_STATEMENT_TYPES: StatementType[] = [StatementType.Create, StatementType.Declare, StatementType.Set, StatementType.Delete]; +const SINGLE_LINE_STATEMENT_TYPES: StatementType[] = [StatementType.Create, StatementType.Declare, StatementType.Set, StatementType.Delete, StatementType.Call]; export function formatSql(textDocument: string, options: FormatOptions = {}): string { let result: string[] = []; @@ -22,6 +23,7 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st for (const statementGroup of statementGroups) { let currentIndent = 0; + let prevType = statementGroup.statements[0].type; for (let i = 0; i < statementGroup.statements.length; i++) { const statement = statementGroup.statements[i]; const withBlocks = SQLTokeniser.createBlocks(statement.tokens); @@ -30,6 +32,12 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st currentIndent -= 4; } + if (options.spaceBetweenStatements) { + if (prevType !== statement.type) { + result.push(``); + } + } + result.push(...formatTokens(withBlocks, options).map(l => ``.padEnd(currentIndent) + l)); if (!statement.isCompoundStart()) { result[result.length-1] += `;` @@ -38,6 +46,8 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st if (statement.isCompoundStart() || statement.isConditionStart()) { currentIndent += 4; } + + prevType = statement.type; } } diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts index 7e1ef625..d5903bfa 100644 --- a/src/language/sql/tests/format.test.ts +++ b/src/language/sql/tests/format.test.ts @@ -7,14 +7,16 @@ const optionsUpper: FormatOptions = { indentWidth: 4, keywordCase: 'upper', identifierCase: 'upper', - newLineLists: true + newLineLists: true, + spaceBetweenStatements: true } const optionsLower: FormatOptions = { indentWidth: 4, keywordCase: 'lower', identifierCase: 'lower', - newLineLists: true + newLineLists: true, + spaceBetweenStatements: true } @@ -137,10 +139,57 @@ test(`CREATE FUNCTION: with single parameter`, () => { `create function "TestDelimiters"."Delimited Function"(`, ` "Delimited Parameter" integer`, `) returns integer language sql begin`, + ``, ` return "Delimited Parameter";`, + ``, `end;` ].join(`\n`)); -}) +}); + +test(`CREATE PROCEDURE: with complex body`, () => { + const sql = [ + `create or replace procedure liama.sql_system(IN command char(512), in curlib varchar(10) default '*SAME', IN libl varchar(512) default '*SAME')`, + ` program type sub`, + ` result sets 1`, + `begin`, + ` declare startTime timestamp;`, + ` declare endTime timestamp;`, + ``, + ` declare theJob varchar(28);`, + ` declare spool_name varchar(10);`, + ` declare spool_number int;`, + ` declare chgcurlib varchar(1024);`, + ``, + ` declare c_result CURSOR FOR`, + ` select SPOOLED_DATA from `, + ` table(systools.spooled_file_data(theJob, spooled_file_name => spool_name, spooled_file_number => spool_number));`, + ``, + ` set chgcurlib = 'chglibl curlib(' concat curlib concat ') libl(' concat libl concat ')';`, + ``, + ` if (curlib <> '*SAME' or libl <> '*SAME') then`, + ` call qsys2.qcmdexc(chgcurlib);`, + ` end if;`, + ``, + ` set startTime = current_timestamp;`, + ``, + ` call qsys2.qcmdexc(command);`, + ``, + ` set endTime = current_timestamp;`, + ``, + ` select `, + ` char(job_number) || '/' || job_user || '/' || job_name, `, + ` spooled_file_name, `, + ` spooled_file_number `, + ` into theJob, spool_name, spool_number`, + ` from table(qsys2.spooled_file_info(starting_timestamp => startTime, ending_timestamp => endTime)) x order by creation_timestamp desc limit 1;`, + ``, + ` open c_result;`, + `end;`, + ].join(`\n`); + + const formatted = formatSql(sql, optionsUpper); + console.log(formatted); +}); test('Active jobs (from Nav)', () => { const sql = `SELECT * FROM TABLE ( QSYS2.ACTIVE_JOB_INFO( RESET_STATISTICS => 'NO', SUBSYSTEM_LIST_FILTER => '', JOB_NAME_FILTER => '*ALL', CURRENT_USER_LIST_FILTER => '', DETAILED_INFO => 'NONE' ) ) ORDER BY SUBSYSTEM, RUN_PRIORITY, JOB_NAME_SHORT, JOB_NUMBER LIMIT 100 OFFSET 0`; diff --git a/src/language/sql/tokens.ts b/src/language/sql/tokens.ts index e9553f32..39267d75 100644 --- a/src/language/sql/tokens.ts +++ b/src/language/sql/tokens.ts @@ -106,6 +106,11 @@ export default class SQLTokeniser { match: [{ type: `equals` }, { type: `morethan` }], becomes: `rightpipe`, }, + { + name: `NOT`, + match: [{type: `lessthan`}, {type: `morethan`}], + becomes: `not` + } ]; readonly spaces = [`\t`, ` `]; readonly splitParts: string[] = [`(`, `)`, `/`, `.`, `*`, `-`, `+`, `;`, `"`, `&`, `%`, `,`, `|`, `?`, `:`, `=`, `<`, `>`, `\n`, `\r`, ...this.spaces]; From 296a2514dbf9a4d56e71f0bd2570f59f71feaa81 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Sat, 21 Sep 2024 15:24:24 -0400 Subject: [PATCH 22/28] Formatter settings Signed-off-by: worksofliam --- src/contributes.json | 22 +++++++++++----------- src/language/providers/formatProvider.ts | 7 ++++--- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/contributes.json b/src/contributes.json index 623eb652..7ccc2217 100644 --- a/src/contributes.json +++ b/src/contributes.json @@ -30,30 +30,30 @@ "properties": { "vscode-db2i.sqlFormat.identifierCase": { "type": "string", - "description": "SQL identifiers", + "description": "Format SQL identifiers into a certain case", "default": "preserve", "enum": [ "lower", "upper", "preserve" - ], - "enumDescriptions": [ - "Format SQL identifiers in lowercase", - "Format SQL identifiers in uppercase", - "Preserve the current formatting of SQL identifiers" ] }, "vscode-db2i.sqlFormat.keywordCase": { "type": "string", - "description": "SQL keywords", + "description": "Format SQL keywords into a certain case", "default": "lower", "enum": [ "lower", "upper" - ], - "enumDescriptions": [ - "Format reserved SQL keywords in lowercase", - "Format reserved SQL keywords in uppercase" + ] + }, + "vscode-db2i.sqlFormat.spaceBetweenStatements": { + "type": "string", + "description": "Add space between statements.", + "default": "true", + "enum": [ + "true", + "false" ] } } diff --git a/src/language/providers/formatProvider.ts b/src/language/providers/formatProvider.ts index 5eb03eef..af0350d7 100644 --- a/src/language/providers/formatProvider.ts +++ b/src/language/providers/formatProvider.ts @@ -7,13 +7,14 @@ export const formatProvider = languages.registerDocumentFormattingEditProvider({ async provideDocumentFormattingEdits(document, options, token) { const identifierCase: CaseOptions = (Configuration.get(`sqlFormat.identifierCase`) || `preserve`); const keywordCase: CaseOptions = (Configuration.get(`sqlFormat.keywordCase`) || `lower`); + const spaceBetweenStatements: string = (Configuration.get(`sqlFormat.spaceBetweenStatements`) || `false`); const formatted = formatSql( document.getText(), { indentWidth: options.tabSize, - identifierCase: identifierCase, - keywordCase: keywordCase, - spaceBetweenStatements: true + identifierCase, + keywordCase, + spaceBetweenStatements: spaceBetweenStatements === `true` } ); From 258e6ebf44453fdce0255f37d3f11171b560d636 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Sun, 22 Sep 2024 04:33:00 -0400 Subject: [PATCH 23/28] Support for references to objects in column list Signed-off-by: worksofliam --- package.json | 22 +++--- src/language/sql/document.ts | 21 ++++-- src/language/sql/formatter.ts | 2 +- src/language/sql/statement.ts | 61 ++++++++++------ src/language/sql/tests/format.test.ts | 1 - src/language/sql/tests/statements.test.ts | 88 +++++++++++++++++------ src/language/sql/tokens.ts | 2 +- src/language/sql/types.ts | 6 +- 8 files changed, 136 insertions(+), 67 deletions(-) diff --git a/package.json b/package.json index 17f9fa2a..87772764 100644 --- a/package.json +++ b/package.json @@ -100,30 +100,30 @@ "properties": { "vscode-db2i.sqlFormat.identifierCase": { "type": "string", - "description": "SQL identifiers", + "description": "Format SQL identifiers into a certain case", "default": "preserve", "enum": [ "lower", "upper", "preserve" - ], - "enumDescriptions": [ - "Format SQL identifiers in lowercase", - "Format SQL identifiers in uppercase", - "Preserve the current formatting of SQL identifiers" ] }, "vscode-db2i.sqlFormat.keywordCase": { "type": "string", - "description": "SQL keywords", + "description": "Format SQL keywords into a certain case", "default": "lower", "enum": [ "lower", "upper" - ], - "enumDescriptions": [ - "Format reserved SQL keywords in lowercase", - "Format reserved SQL keywords in uppercase" + ] + }, + "vscode-db2i.sqlFormat.spaceBetweenStatements": { + "type": "string", + "description": "Add space between statements.", + "default": "true", + "enum": [ + "true", + "false" ] } } diff --git a/src/language/sql/document.ts b/src/language/sql/document.ts index 34e1c533..6664035e 100644 --- a/src/language/sql/document.ts +++ b/src/language/sql/document.ts @@ -35,6 +35,7 @@ export default class Document { let statementStart = 0; for (let i = 0; i < tokens.length; i++) { + const upperValue = tokens[i].value?.toUpperCase(); switch (tokens[i].type) { case `semicolon`: const statementTokens = tokens.slice(statementStart, i); @@ -45,19 +46,25 @@ export default class Document { break; case `statementType`: - currentStatementType = StatementTypeWord[tokens[i].value?.toUpperCase()]; + currentStatementType = StatementTypeWord[upperValue]; break; case `keyword`: - switch (tokens[i].value?.toUpperCase()) { + switch (upperValue) { case `LOOP`: - // This handles the case that 'END LOOP' is supported. - if (currentStatementType === StatementType.End) { - break; - } + case `THEN`: case `BEGIN`: case `DO`: - case `THEN`: + // This handles the case that 'END LOOP' is supported. + if (upperValue === `LOOP` && currentStatementType === StatementType.End) { + break; + } + + // Support for THEN in conditionals + if (upperValue === `THEN` && !Statement.typeIsConditional(currentStatementType)) { + break; + } + // We include BEGIN in the current statement // then the next statement beings const statementTokens = tokens.slice(statementStart, i+1); diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 00c6e4b8..501e6bc3 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -12,7 +12,7 @@ export interface FormatOptions { spaceBetweenStatements?: boolean } -const SINGLE_LINE_STATEMENT_TYPES: StatementType[] = [StatementType.Create, StatementType.Declare, StatementType.Set, StatementType.Delete, StatementType.Call]; +const SINGLE_LINE_STATEMENT_TYPES: StatementType[] = [StatementType.Create, StatementType.Declare, StatementType.Set, StatementType.Delete, StatementType.Call, StatementType.If, StatementType.End]; export function formatSql(textDocument: string, options: FormatOptions = {}): string { let result: string[] = []; diff --git a/src/language/sql/statement.ts b/src/language/sql/statement.ts index d06a00f1..a9a14182 100644 --- a/src/language/sql/statement.ts +++ b/src/language/sql/statement.ts @@ -56,8 +56,12 @@ export default class Statement { return false; } + static typeIsConditional(type: StatementType) { + return [StatementType.If, StatementType.While, StatementType.Loop, StatementType.For].includes(type); + } + isConditionStart() { - return [StatementType.If, StatementType.While, StatementType.Loop, StatementType.For, StatementType.Case, StatementType.When].includes(this.type); + return Statement.typeIsConditional(this.type); } isConditionEnd() { @@ -327,17 +331,21 @@ export default class Statement { } const basicQueryFinder = (startIndex: number): void => { + let currentClause: undefined|"select"|"from"; for (let i = startIndex; i < this.tokens.length; i++) { if (tokenIs(this.tokens[i], `clause`, `FROM`)) { - inFromClause = true; - } else if (inFromClause && tokenIs(this.tokens[i], `clause`) || tokenIs(this.tokens[i], `join`) || tokenIs(this.tokens[i], `closebracket`)) { - inFromClause = false; + currentClause = `from`; + } + else if (tokenIs(this.tokens[i], `statementType`, `SELECT`)) { + currentClause = `select`; + } else if (currentClause === `from` && tokenIs(this.tokens[i], `clause`) || tokenIs(this.tokens[i], `join`) || tokenIs(this.tokens[i], `closebracket`)) { + currentClause = undefined; } if (tokenIs(this.tokens[i], `clause`, `FROM`) || (this.type !== StatementType.Select && tokenIs(this.tokens[i], `clause`, `INTO`)) || tokenIs(this.tokens[i], `join`) || - (inFromClause && tokenIs(this.tokens[i], `comma`) + (currentClause === `from` && tokenIs(this.tokens[i], `comma`) )) { const sqlObj = this.getRefAtToken(i+1); if (sqlObj) { @@ -347,6 +355,15 @@ export default class Statement { i += 3; //For the brackets } } + } else if (currentClause === `select` && tokenIs(this.tokens[i], `function`)) { + const sqlObj = this.getRefAtToken(i); + if (sqlObj) { + doAdd(sqlObj); + i += sqlObj.tokens.length; + if (sqlObj.isUDTF || sqlObj.fromLateral) { + i += 3; //For the brackets + } + } } } } @@ -605,7 +622,7 @@ export default class Statement { } if (options.withSystemName) { - if (tokenIs(this.tokens[endIndex+1], `keyword`, `FOR`) && tokenIs(this.tokens[endIndex+2], `word`, `SYSTEM`) && tokenIs(this.tokens[endIndex+3], `word`, `NAME`)) { + if (tokenIs(this.tokens[endIndex+1], `statementType`, `FOR`) && tokenIs(this.tokens[endIndex+2], `word`, `SYSTEM`) && tokenIs(this.tokens[endIndex+3], `word`, `NAME`)) { if (this.tokens[endIndex+4] && NameTypes.includes(this.tokens[endIndex+4].type)) { sqlObj.object.system = this.tokens[endIndex+4].value; } @@ -637,10 +654,25 @@ export default class Statement { switch (currentToken.type) { case `statementType`: - if (declareStmt) continue; + const currentValue = currentToken.value.toLowerCase(); + if (declareStmt) { + if (currentValue === `for`) { + ranges.push({ + type: `remove`, + range: { + start: declareStmt.range.start, + end: currentToken.range.end + } + }); + + declareStmt = undefined; + } + + continue; + }; // If we're in a DECLARE, it's likely a cursor definition - if (currentToken.value.toLowerCase() === `declare`) { + if (currentValue === `declare`) { declareStmt = currentToken; } break; @@ -729,19 +761,6 @@ export default class Statement { } }); } - } else - if (declareStmt && tokenIs(currentToken, `keyword`, `FOR`)) { - // If we're a DECLARE, and we found the FOR keyword, the next - // set of tokens should be the select. - ranges.push({ - type: `remove`, - range: { - start: declareStmt.range.start, - end: currentToken.range.end - } - }); - - declareStmt = undefined; } break; } diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts index d5903bfa..f8d9ec95 100644 --- a/src/language/sql/tests/format.test.ts +++ b/src/language/sql/tests/format.test.ts @@ -188,7 +188,6 @@ test(`CREATE PROCEDURE: with complex body`, () => { ].join(`\n`); const formatted = formatSql(sql, optionsUpper); - console.log(formatted); }); test('Active jobs (from Nav)', () => { diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts index 023195df..f6e514d2 100644 --- a/src/language/sql/tests/statements.test.ts +++ b/src/language/sql/tests/statements.test.ts @@ -205,7 +205,7 @@ parserScenarios(`Object references`, ({newDoc}) => { const statement = document.statements[0]; const refs = statement.getObjectReferences(); - expect(refs.length).toBe(3); + expect(refs.length).toBe(5); expect(statement.getClauseForOffset(10)).toBe(ClauseType.Unknown); expect(statement.getClauseForOffset(125)).toBe(ClauseType.From); @@ -835,9 +835,11 @@ parserScenarios(`Object references`, ({newDoc}) => { const selectStatement = group.statements[2]; expect(selectStatement.type).toBe(StatementType.Select); const refsC = selectStatement.getObjectReferences(); - expect(refsC.length).toBe(1); + expect(refsC.length).toBe(2); expect(refsC[0].createType).toBeUndefined(); - expect(refsC[0].object.name).toBe(`employee`); + expect(refsC[0].object.name).toBe(`sum`); + expect(refsC[1].createType).toBeUndefined(); + expect(refsC[1].object.name).toBe(`employee`); }); test(`CREATE FUNCTION: with multiple parameters`, () => { @@ -1291,27 +1293,36 @@ parserScenarios(`PL body tests`, ({newDoc}) => { const refs = statement.getObjectReferences(); const ctes = statement.getCTEReferences(); - expect(refs.length).toBe(7); + expect(refs.length).toBe(10); expect(refs[0].object.name).toBe(`shipments`); expect(refs[0].alias).toBe(`s`); expect(refs[1].object.name).toBe(`BillingDate`); expect(refs[1].alias).toBeUndefined(); - expect(refs[2].object.name).toBe(`Temp01`); - expect(refs[2].alias).toBe(`t1`); + expect(refs[2].object.name).toBe(`sum`); + expect(refs[2].alias).toBeUndefined(); expect(refs[3].object.name).toBe(`Temp01`); expect(refs[3].alias).toBe(`t1`); - expect(refs[4].object.name).toBe(`Temp02`); - expect(refs[4].alias).toBe(`t2`); + expect(refs[4].object.name).toBe(`dec`); + expect(refs[4].alias).toBeUndefined(); + + expect(refs[5].object.name).toBe(`round`); + expect(refs[5].alias).toBeUndefined(); + + expect(refs[6].object.name).toBe(`Temp01`); + expect(refs[6].alias).toBe(`t1`); - expect(refs[5].object.name).toBe(`customers`); - expect(refs[5].alias).toBe(`c`); + expect(refs[7].object.name).toBe(`Temp02`); + expect(refs[7].alias).toBe(`t2`); - expect(refs[6].object.name).toBe(`Temp02`); - expect(refs[6].alias).toBeUndefined(); + expect(refs[8].object.name).toBe(`customers`); + expect(refs[8].alias).toBe(`c`); + + expect(refs[9].object.name).toBe(`Temp02`); + expect(refs[9].alias).toBeUndefined(); expect(ctes.length).toBe(3); expect(ctes[0].name).toBe(`Temp01`); @@ -1322,11 +1333,14 @@ parserScenarios(`PL body tests`, ({newDoc}) => { expect(ctes[2].name).toBe(`Temp03`); expect(ctes[2].columns.length).toBe(0); + const temp03Stmt = ctes[2].statement.getObjectReferences(); - expect(temp03Stmt.length).toBe(3); - expect(temp03Stmt[0].object.name).toBe(`Temp01`); - expect(temp03Stmt[1].object.name).toBe(`Temp02`); - expect(temp03Stmt[2].object.name).toBe(`customers`); + console.log(temp03Stmt); + + expect(temp03Stmt[0].object.name).toBe(`dec`); + expect(temp03Stmt[1].object.name).toBe(`Temp01`); + expect(temp03Stmt[2].object.name).toBe(`Temp02`); + expect(temp03Stmt[3].object.name).toBe(`customers`); }) test(`WITH: explicit columns`, () => { @@ -1439,10 +1453,44 @@ parserScenarios(`PL body tests`, ({newDoc}) => { expect(statement.type).toBe(StatementType.Select); const objs = statement.getObjectReferences(); - expect(objs.length).toBe(1); - expect(objs[0].object.schema).toBe(`qsys2`); - expect(objs[0].object.name).toBe(`ACTIVE_JOB_INFO`); - }) + + console.log(objs); + expect(objs.length).toBe(2); + expect(objs[1].object.schema).toBe(`qsys2`); + expect(objs[1].object.name).toBe(`ACTIVE_JOB_INFO`); + }); + + test('CASE, WHEN, END', () => { + const lines = [ + `--`, + ``, + `--`, + `-- Hold any jobs that started running an SQL statement more than 2 hours ago.`, + `--`, + `select JOB_NAME,`, + ` case`, + ` when QSYS2.QCMDEXC('HLDJOB ' concat JOB_NAME) = 1 then 'Job Held'`, + ` else 'Job not held'`, + ` end as HLDJOB_RESULT`, + ` from table (`, + ` QSYS2.ACTIVE_JOB_INFO(DETAILED_INFO => 'ALL')`, + ` )`, + ` where SQL_STATEMENT_START_TIMESTAMP < current timestamp - 2 hours;`, + ].join(`\n`); + + const document = newDoc(lines); + + + const statements = document.statements; + expect(statements.length).toBe(1); + + const statement = statements[0]; + expect(statement.type).toBe(StatementType.Select); + + const objs = statement.getObjectReferences(); + + expect(objs.length).toBe(2); + }); }); describe(`Parameter statement tests`, () => { diff --git a/src/language/sql/tokens.ts b/src/language/sql/tokens.ts index 39267d75..dc153c90 100644 --- a/src/language/sql/tokens.ts +++ b/src/language/sql/tokens.ts @@ -34,7 +34,7 @@ export default class SQLTokeniser { { name: `STATEMENTTYPE`, match: [{ type: `word`, match: (value: string) => { - return [`CREATE`, `ALTER`, `SELECT`, `WITH`, `INSERT`, `UPDATE`, `DELETE`, `DROP`, `CALL`, `DECLARE`].includes(value.toUpperCase()); + return [`CREATE`, `ALTER`, `SELECT`, `WITH`, `INSERT`, `UPDATE`, `DELETE`, `DROP`, `CALL`, `DECLARE`, `IF`, `FOR`, `WHILE`].includes(value.toUpperCase()); } }], becomes: `statementType`, }, diff --git a/src/language/sql/types.ts b/src/language/sql/types.ts index c81b7349..0084b6e9 100644 --- a/src/language/sql/types.ts +++ b/src/language/sql/types.ts @@ -17,7 +17,6 @@ export enum StatementType { Elseif = "Elseif", Call = "Call", Alter = "Alter", - Case = "Case", Fetch = "Fetch", For = "For", Get = "Get", @@ -34,8 +33,7 @@ export enum StatementType { Return = "Return", Signal = "Signal", Set = "Set", - While = "While", - When = "When" + While = "While" } export const StatementTypeWord = { @@ -53,7 +51,6 @@ export const StatementTypeWord = { 'CALL': StatementType.Call, 'BEGIN': StatementType.Begin, 'ALTER': StatementType.Alter, - 'CASE': StatementType.Case, 'FOR': StatementType.For, 'FETCH': StatementType.Fetch, 'GET': StatementType.Get, @@ -70,7 +67,6 @@ export const StatementTypeWord = { 'SIGNAL': StatementType.Signal, 'SET': StatementType.Set, 'WHILE': StatementType.While, - 'WHEN': StatementType.When, }; export enum ClauseType { From d6e287ebfc3d270caa81ee2c64da2a8a15025841 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Sun, 22 Sep 2024 04:34:26 -0400 Subject: [PATCH 24/28] Remove logs Signed-off-by: worksofliam --- src/language/sql/tests/statements.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts index f6e514d2..dc7d977c 100644 --- a/src/language/sql/tests/statements.test.ts +++ b/src/language/sql/tests/statements.test.ts @@ -1335,7 +1335,6 @@ parserScenarios(`PL body tests`, ({newDoc}) => { expect(ctes[2].columns.length).toBe(0); const temp03Stmt = ctes[2].statement.getObjectReferences(); - console.log(temp03Stmt); expect(temp03Stmt[0].object.name).toBe(`dec`); expect(temp03Stmt[1].object.name).toBe(`Temp01`); @@ -1454,7 +1453,6 @@ parserScenarios(`PL body tests`, ({newDoc}) => { const objs = statement.getObjectReferences(); - console.log(objs); expect(objs.length).toBe(2); expect(objs[1].object.schema).toBe(`qsys2`); expect(objs[1].object.name).toBe(`ACTIVE_JOB_INFO`); From 008e435608baaf1bdbb30a47f54845b4c96095ad Mon Sep 17 00:00:00 2001 From: worksofliam Date: Mon, 23 Sep 2024 02:11:08 -0400 Subject: [PATCH 25/28] Improvements to show a signature Signed-off-by: worksofliam --- src/language/providers/hoverProvider.ts | 23 +++++++++++++++-------- src/language/sql/formatter.ts | 2 +- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/language/providers/hoverProvider.ts b/src/language/providers/hoverProvider.ts index bfab1495..714f249f 100644 --- a/src/language/providers/hoverProvider.ts +++ b/src/language/providers/hoverProvider.ts @@ -84,14 +84,21 @@ export const hoverProvider = languages.registerHoverProvider({ language: `sql` } if (result) { if ('routine' in result) { const routineOffset = ref.tokens[ref.tokens.length-1].range.end+1; - const callableRef = statementAt.getCallableDetail(routineOffset, false); - if (callableRef) { - const { currentCount } = getPositionData(callableRef, routineOffset); - const signatures = await DbCache.getCachedSignatures(callableRef.parentRef.object.schema, callableRef.parentRef.object.name); - const possibleSignatures = signatures.filter((s) => s.parms.length >= currentCount).sort((a, b) => a.parms.length - b.parms.length); - const signature = possibleSignatures.find((signature) => currentCount <= signature.parms.length); - if (signature) { - addRoutineMd(md, signature, result); + const callableRef = statementAt.getCallableDetail(routineOffset, false) + const signatures = await DbCache.getCachedSignatures(schema, ref.object.name); + + if (signatures.length > 0) { + let chosenSignature: CallableSignature | undefined; + if (callableRef) { + const { currentCount } = getPositionData(callableRef, routineOffset); + const possibleSignatures = signatures.filter((s) => s.parms.length >= currentCount).sort((a, b) => a.parms.length - b.parms.length); + chosenSignature = possibleSignatures.find((signature) => currentCount <= signature.parms.length); + } else { + chosenSignature = signatures[0]; + } + + if (chosenSignature) { + addRoutineMd(md, chosenSignature, result); } } } else { diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 501e6bc3..8fddf575 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -20,10 +20,10 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st const statementGroups: StatementGroup[] = document.getStatementGroups(); const eol = textDocument.includes(`\r\n`) ? `\r\n` : `\n`; + let prevType = StatementType.Unknown; for (const statementGroup of statementGroups) { let currentIndent = 0; - let prevType = statementGroup.statements[0].type; for (let i = 0; i < statementGroup.statements.length; i++) { const statement = statementGroup.statements[i]; const withBlocks = SQLTokeniser.createBlocks(statement.tokens); From 6c3ee2972470e50b5913be63e4b54efef9f40e78 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Mon, 23 Sep 2024 02:19:40 -0400 Subject: [PATCH 26/28] Fix test cases Signed-off-by: worksofliam --- src/language/sql/formatter.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/language/sql/formatter.ts b/src/language/sql/formatter.ts index 8fddf575..cdcd4f80 100644 --- a/src/language/sql/formatter.ts +++ b/src/language/sql/formatter.ts @@ -32,17 +32,17 @@ export function formatSql(textDocument: string, options: FormatOptions = {}): st currentIndent -= 4; } - if (options.spaceBetweenStatements) { - if (prevType !== statement.type) { - result.push(``); - } - } - result.push(...formatTokens(withBlocks, options).map(l => ``.padEnd(currentIndent) + l)); if (!statement.isCompoundStart()) { result[result.length-1] += `;` } + if (options.spaceBetweenStatements) { + if (prevType !== statement.type && i < statementGroup.statements.length - 1) { + result.push(``); + } + } + if (statement.isCompoundStart() || statement.isConditionStart()) { currentIndent += 4; } From 04b43ebb7ff7340d4b13b38f6422c5822167d2e9 Mon Sep 17 00:00:00 2001 From: Julia Yan Date: Mon, 23 Sep 2024 17:01:17 -0400 Subject: [PATCH 27/28] Added test cases --- src/language/sql/tests/format.test.ts | 236 ++++++++++++++++++-------- 1 file changed, 168 insertions(+), 68 deletions(-) diff --git a/src/language/sql/tests/format.test.ts b/src/language/sql/tests/format.test.ts index f8d9ec95..2a8ab79f 100644 --- a/src/language/sql/tests/format.test.ts +++ b/src/language/sql/tests/format.test.ts @@ -19,112 +19,135 @@ const optionsLower: FormatOptions = { spaceBetweenStatements: true } +const optionsNoNewLine: FormatOptions = { + indentWidth: 4, + keywordCase: 'lower', + identifierCase: 'lower', + newLineLists: false, + spaceBetweenStatements: true +} + // Edit an assertion and save to see HMR in action test('Clause new line - upper', () => { const sql = `select * from sample`; const formatted = formatSql(sql, optionsUpper); - expect(formatted).toBe(`SELECT - * -FROM - SAMPLE;`); + expect(formatted).toBe([ + `SELECT`, + ` *`, + `FROM`, + ` SAMPLE;`, + ].join(`\n`)); }); test('Clause new line - lower', () => { const sql = `select * from sample`; const formatted = formatSql(sql, optionsLower); - expect(formatted).toBe(`select - * -from - sample;`); + expect(formatted).toBe([ + `select`, + ` *`, + `from`, + ` sample;`, + ].join(`\n`)); }); test('Two clause statements', () => { const sql = `select * from sample;\nselect * from sample;`; const formatted = formatSql(sql, optionsUpper); - expect(formatted).toBe(`SELECT - * -FROM - SAMPLE; -SELECT - * -FROM - SAMPLE;`); + expect(formatted).toBe([ + `SELECT`, + ` *`, + `FROM`, + ` SAMPLE;`, + `SELECT`, + ` *`, + `FROM`, + ` SAMPLE;`, + ].join(`\n`)); }); test('Simple multi clause', () => { const sql = `select * from sample limit 1`; const formatted = formatSql(sql, optionsUpper); - expect(formatted).toBe( -`SELECT - * -FROM - SAMPLE -LIMIT - 1;`); + expect(formatted).toBe([ + `SELECT`, + ` *`, + `FROM`, + ` SAMPLE`, + `LIMIT`, + ` 1;`, + ].join(`\n`)); }); test('Brackets', () => { const sql = `SELECT * FROM SAMPLE(RESET_STATISTICS => 'NO', JOB_NAME_FILTER => '*ALL', DETAILED_INFO => 'NONE') WHERE UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' ORDER BY JOB_NAME_SHORT, JOB_NUMBER;`; const formatted = formatSql(sql, optionsUpper); - expect(formatted).toBe(`SELECT - * -FROM - SAMPLE( - RESET_STATISTICS => 'NO', - JOB_NAME_FILTER => '*ALL', - DETAILED_INFO => 'NONE' - ) -WHERE - UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' -ORDER BY - JOB_NAME_SHORT, - JOB_NUMBER;`); + expect(formatted).toBe([ + `SELECT`, + ` *`, + `FROM`, + ` SAMPLE(`, + ` RESET_STATISTICS => 'NO',`, + ` JOB_NAME_FILTER => '*ALL',`, + ` DETAILED_INFO => 'NONE'`, + ` )`, + `WHERE`, + ` UPPER(JOB_NAME) LIKE '%QNAVMNSRV%'`, + `ORDER BY`, + ` JOB_NAME_SHORT,`, + ` JOB_NUMBER;`, + ].join(`\n`)); }); test('Select with columns', () => { const sql = `SELECT ONE, TWO, THREE FROM SAMPLE2`; const formatted = formatSql(sql, optionsUpper); - expect(formatted).toBe(`SELECT - ONE, - TWO, - THREE -FROM - SAMPLE2;`); + expect(formatted).toBe([ + `SELECT`, + ` ONE,`, + ` TWO,`, + ` THREE`, + `FROM`, + ` SAMPLE2;` + ].join(`\n`)); }); test('Nested Select', () => { const sql = `SELECT * FROM SAMPLE ( SELECT ONE, TWO, THREE FROM SAMPLE2 ) WHERE UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' ORDER BY JOB_NAME_SHORT, JOB_NUMBER;`; const formatted = formatSql(sql, optionsUpper); - expect(formatted).toBe(`SELECT - * -FROM - SAMPLE ( - SELECT - ONE, - TWO, - THREE - FROM - SAMPLE2 - ) -WHERE - UPPER(JOB_NAME) LIKE '%QNAVMNSRV%' -ORDER BY - JOB_NAME_SHORT, - JOB_NUMBER;`); + expect(formatted).toBe([ + `SELECT`, + ` *`, + `FROM`, + ` SAMPLE (`, + ` SELECT`, + ` ONE,`, + ` TWO,`, + ` THREE`, + ` FROM`, + ` SAMPLE2`, + ` )`, + `WHERE`, + ` UPPER(JOB_NAME) LIKE '%QNAVMNSRV%'`, + `ORDER BY`, + ` JOB_NAME_SHORT,`, + ` JOB_NUMBER;` + ].join(`\n`)); }); test('Alter Table to Add Materialized Query (from ACS)', () => { const sql = `ALTER TABLE table1 ADD MATERIALIZED QUERY (SELECT int_col, varchar_col FROM table3) DATA INITIALLY IMMEDIATE REFRESH DEFERRED MAINTAINED BY USER ENABLE QUERY OPTIMIZATION;`; const formatted = formatSql(sql, optionsUpper); - expect(formatted).toBe(`ALTER - TABLE TABLE1 ADD MATERIALIZED QUERY ( - SELECT - INT_COL, - VARCHAR_COL - FROM - TABLE3 - ) DATA INITIALLY IMMEDIATE REFRESH DEFERRED MAINTAINED BY USER ENABLE QUERY OPTIMIZATION;`); + expect(formatted).toBe([ + `ALTER`, + ` TABLE TABLE1 ADD MATERIALIZED QUERY (`, + ` SELECT`, + ` INT_COL,`, + ` VARCHAR_COL`, + ` FROM`, + ` TABLE3`, + ` ) DATA INITIALLY IMMEDIATE REFRESH DEFERRED MAINTAINED BY USER ENABLE QUERY OPTIMIZATION;`, + ].join(`\n`)); }); test(`CREATE FUNCTION: with single parameter`, () => { @@ -193,7 +216,84 @@ test(`CREATE PROCEDURE: with complex body`, () => { test('Active jobs (from Nav)', () => { const sql = `SELECT * FROM TABLE ( QSYS2.ACTIVE_JOB_INFO( RESET_STATISTICS => 'NO', SUBSYSTEM_LIST_FILTER => '', JOB_NAME_FILTER => '*ALL', CURRENT_USER_LIST_FILTER => '', DETAILED_INFO => 'NONE' ) ) ORDER BY SUBSYSTEM, RUN_PRIORITY, JOB_NAME_SHORT, JOB_NUMBER LIMIT 100 OFFSET 0`; const formatted = formatSql(sql, optionsUpper); - // console.log('*************'); + expect(formatted).toBe([ + `SELECT`, + ` *`, + `FROM`, + ` TABLE (`, + ` QSYS2.ACTIVE_JOB_INFO(`, + ` RESET_STATISTICS => 'NO',`, + ` SUBSYSTEM_LIST_FILTER => '',`, + ` JOB_NAME_FILTER => '*ALL',`, + ` CURRENT_USER_LIST_FILTER => '',`, + ` DETAILED_INFO => 'NONE'`, + ` )`, + ` )`, + `ORDER BY`, + ` SUBSYSTEM,`, + ` RUN_PRIORITY,`, + ` JOB_NAME_SHORT,`, + ` JOB_NUMBER`, + `LIMIT`, + ` 100`, + `OFFSET`, + ` 0;`, + ].join(`\n`)); +}); + +test('Select WITH', () => { + const sql = `WITH A AS ( SELECT * FROM TABLE ( QSYS2.IFS_OBJECT_STATISTICS( START_PATH_NAME => '/QIBM/ProdData/HTTPA/admin/www/', SUBTREE_DIRECTORIES => 'NO', OBJECT_TYPE_LIST => '', OMIT_LIST => '', IGNORE_ERRORS => 'YES' ) ) ) SELECT * FROM A WHERE UPPER(PATH_NAME) LIKE '%HTML%' ORDER BY UPPER(PATH_NAME) ASC LIMIT 500 OFFSET 0`; + const formatted = formatSql(sql, optionsUpper); + expect(formatted).toBe([ + `WITH`, + ` A AS (`, + ` SELECT`, + ` *`, + ` FROM`, + ` TABLE (`, + ` QSYS2.IFS_OBJECT_STATISTICS(`, + ` START_PATH_NAME => '/QIBM/PRODDATA/HTTPA/ADMIN/WWW/',`, + ` SUBTREE_DIRECTORIES => 'NO',`, + ` OBJECT_TYPE_LIST => '',`, + ` OMIT_LIST => '',`, + ` IGNORE_ERRORS => 'YES'`, + ` )`, + ` )`, + ` )`, + `SELECT`, + ` *`, + `FROM`, + ` A`, + `WHERE`, + ` UPPER(PATH_NAME) LIKE '%HTML%'`, + `ORDER BY`, + ` UPPER(PATH_NAME) ASC`, + `LIMIT`, + ` 500`, + `OFFSET`, + ` 0;`, + ].join(`\n`)); +}); + +test('Create and Insert', () => { + const sql = [ + `CREATE TABLE emp(name VARCHAR(100) CCSID 1208, id int);`, + `INSERT INTO emp VALUES ('name', 1);` + ].join(`\n`); + const formatted = formatSql(sql, optionsUpper); + // console.log("*******"); // console.log(formatted); - // console.log('*************'); + // console.log("*******"); + expect(formatted).toBe([ + `CREATE TABLE EMP (`, + ` NAME VARCHAR(100) CCSID 1208,`, + ` ID INT`, + `);`, + `INSERT INTO`, + ` EMP`, + `VALUES(`, + ` 'NAME',`, + ` 1`, + `);`, + ].join(`\n`)); }); \ No newline at end of file From 9ae112bc810521a0d79bdd3c59da7df8157c6c1f Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 3 Oct 2024 10:35:40 -0400 Subject: [PATCH 28/28] Additional test for case support Signed-off-by: worksofliam --- src/language/sql/tests/statements.test.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts index dc7d977c..a5c68745 100644 --- a/src/language/sql/tests/statements.test.ts +++ b/src/language/sql/tests/statements.test.ts @@ -1489,6 +1489,28 @@ parserScenarios(`PL body tests`, ({newDoc}) => { expect(objs.length).toBe(2); }); + + + + test('SELECT statement with CASE', () => { + const content = [ + `SELECT`, + ` CLE,`, + ` CASE`, + ` WHEN CLE = 1 THEN 'FIRST' Else VALEUR End As VALEUR`, + `FROM`, + ` QTEMP.Test`, + ].join(` `); + + const document = new Document(content); + const statements = document.statements; + expect(statements.length).toBe(1); + + const statement = statements[0]; + expect(statement.type).toBe(StatementType.Select); + + + }); }); describe(`Parameter statement tests`, () => {