From f7beb75f38c38b065d26f05760438d84224fdb1a Mon Sep 17 00:00:00 2001 From: Pistonight Date: Mon, 16 Dec 2024 19:26:25 -0800 Subject: [PATCH] implement TOML --- src/basic-languages/monaco.contribution.ts | 1 + src/basic-languages/toml/toml.contribution.ts | 25 + src/basic-languages/toml/toml.test.ts | 1027 +++++++++++++++++ src/basic-languages/toml/toml.ts | 410 +++++++ .../website/data/home-samples/sample.toml.txt | 151 +++ 5 files changed, 1614 insertions(+) create mode 100644 src/basic-languages/toml/toml.contribution.ts create mode 100644 src/basic-languages/toml/toml.test.ts create mode 100644 src/basic-languages/toml/toml.ts create mode 100644 website/src/website/data/home-samples/sample.toml.txt diff --git a/src/basic-languages/monaco.contribution.ts b/src/basic-languages/monaco.contribution.ts index d623774265..b86386cfbb 100644 --- a/src/basic-languages/monaco.contribution.ts +++ b/src/basic-languages/monaco.contribution.ts @@ -77,6 +77,7 @@ import './st/st.contribution'; import './swift/swift.contribution'; import './systemverilog/systemverilog.contribution'; import './tcl/tcl.contribution'; +import './toml/toml.contribution'; import './twig/twig.contribution'; import './typescript/typescript.contribution'; import './typespec/typespec.contribution'; diff --git a/src/basic-languages/toml/toml.contribution.ts b/src/basic-languages/toml/toml.contribution.ts new file mode 100644 index 0000000000..059e8e3ce0 --- /dev/null +++ b/src/basic-languages/toml/toml.contribution.ts @@ -0,0 +1,25 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { registerLanguage } from '../_.contribution'; + +declare var AMD: any; +declare var require: any; + +registerLanguage({ + id: 'toml', + extensions: ['.toml'], + aliases: ['TOML', 'toml'], + mimetypes: ['application/toml', 'text/toml'], + loader: () => { + if (AMD) { + return new Promise((resolve, reject) => { + require(['vs/basic-languages/toml/toml'], resolve, reject); + }); + } else { + return import('./toml'); + } + } +}); diff --git a/src/basic-languages/toml/toml.test.ts b/src/basic-languages/toml/toml.test.ts new file mode 100644 index 0000000000..ca49db604f --- /dev/null +++ b/src/basic-languages/toml/toml.test.ts @@ -0,0 +1,1027 @@ +import { type IRelaxedToken, testTokenization } from '../test/testRunner'; + +function makeTokenItemFn(type: string): (startIndex: number) => IRelaxedToken { + return (startIndex: number) => { + return { + startIndex, + type + }; + }; +} + +const white = makeTokenItemFn('white.toml'); +const comment = makeTokenItemFn('comment.toml'); +const string = makeTokenItemFn('string.toml'); +const escape = makeTokenItemFn('constant.character.escape.toml'); +const escapeInvalid = makeTokenItemFn('constant.character.escape.invalid.toml'); +const stringLiteral = makeTokenItemFn('string.literal.toml'); +const stringMulti = makeTokenItemFn('string.multi.toml'); +const stringLiteralMulti = makeTokenItemFn('string.literal.multi.toml'); +const variable = makeTokenItemFn('variable.toml'); +const variableQuoted = makeTokenItemFn('variable.string.toml'); +const variableLiteral = makeTokenItemFn('variable.string.literal.toml'); +const delimiter = makeTokenItemFn('delimiter.toml'); +const square = makeTokenItemFn('delimiter.square.toml'); +const bracket = makeTokenItemFn('delimiter.bracket.toml'); +const boolean = makeTokenItemFn('constant.language.boolean.toml'); +const decimal = makeTokenItemFn('number.toml'); +const octal = makeTokenItemFn('number.octal.toml'); +const hex = makeTokenItemFn('number.hex.toml'); +const binary = makeTokenItemFn('number.binary.toml'); +const float = makeTokenItemFn('number.float.toml'); +const infinity = makeTokenItemFn('number.inf.toml'); +const nan = makeTokenItemFn('number.nan.toml'); +const date = makeTokenItemFn('number.date.toml'); +const time = makeTokenItemFn('number.time.toml'); +const datetime = makeTokenItemFn('number.datetime.toml'); +const table = makeTokenItemFn('type.toml'); +const tableString = makeTokenItemFn('type.string.toml'); +const tableLiteral = makeTokenItemFn('type.string.literal.toml'); + +function testNumber(line: string, fn: (start: number) => any = decimal) { + return { + line, + tokens: [variable(0), white(4), delimiter(5), white(6), fn(7)] + }; +} + +// https://toml.io/en/v1.0.0 +testTokenization('toml', [ + // https://toml.io/en/v1.0.0#comment + [ + { + line: '# comment', + tokens: [comment(0)] + }, + { + line: `another = "# This is not a comment"`, + tokens: [variable(0), white(7), delimiter(8), white(9), string(10)] + } + ], + // https://toml.io/en/v1.0.0#keyvalue-pair + [ + { + line: `key = ""`, + tokens: [variable(0), white(3), delimiter(4), white(5), string(6)] + }, + { + line: 'key= # INVALID', + tokens: [variable(0), delimiter(3), white(4), comment(5)] + }, + { + line: `key = "value"`, + tokens: [variable(0), white(3), delimiter(4), white(5), string(6)] + } + ], + // https://toml.io/en/v1.0.0#keys + [ + { + line: `bare_key = "value"`, + tokens: [variable(0), white(8), delimiter(9), white(10), string(11)] + }, + { + line: `bare-key = "value"`, + tokens: [variable(0), white(8), delimiter(9), white(10), string(11)] + }, + { + line: `1234 = "value"`, + tokens: [variable(0), white(4), delimiter(5), white(6), string(7)] + }, + { + line: `"127.0.0.1" = "value"`, + // ----0123456789 11 + tokens: [variableQuoted(0), white(11), delimiter(12), white(13), string(14)] + }, + { + line: `"character encoding" = "value"`, + // ----0123456789 11 14 17 20 23 26 29 32 + tokens: [variableQuoted(0), white(20), delimiter(21), white(22), string(23)] + }, + { + line: `"ʎǝʞ" = "value"`, + tokens: [variableQuoted(0), white(5), delimiter(6), white(7), string(8)] + }, + { + line: `'key2' = 'value'`, + tokens: [variableLiteral(0), white(6), delimiter(7), white(8), stringLiteral(9)] + }, + { + line: `'quoted "value"' = "value"`, + // ----0123456789 11 14 17 + tokens: [variableLiteral(0), white(16), delimiter(17), white(18), string(19)] + }, + { + line: `= "value" # INVALID`, + // ----012345678 10 + tokens: [delimiter(0), white(1), string(2), white(9), comment(10)] + }, + { + line: `"" = "blank"`, + tokens: [variableQuoted(0), white(2), delimiter(3), white(4), string(5)] + }, + { + line: `"a\\tb" = "blank"`, + // ----012 3456789 + tokens: [ + variableQuoted(0), + escape(2), + variableQuoted(4), + white(6), + delimiter(7), + white(8), + string(9) + ] + }, + { + line: `"a\\mb" = "blank"`, + // ----012 3456789 + tokens: [ + variableQuoted(0), + escapeInvalid(2), + variableQuoted(4), + white(6), + delimiter(7), + white(8), + string(9) + ] + }, + { + line: `'' = 'blank'`, + tokens: [variableLiteral(0), white(2), delimiter(3), white(4), stringLiteral(5)] + }, + { + line: `physical.color = "orange"`, + // ----0123456789 11 14 17 20 23 26 + tokens: [ + variable(0), + delimiter(8), + variable(9), + white(14), + delimiter(15), + white(16), + string(17) + ] + }, + { + line: `site."google.com" = true`, + // ----0123456789 11 14 17 20 + tokens: [ + variable(0), + delimiter(4), + variableQuoted(5), + white(17), + delimiter(18), + white(19), + boolean(20) + ] + }, + { + line: `fruit. color = "yellow"`, + // ----0123456789 11 14 + tokens: [ + variable(0), + delimiter(5), + white(6), + variable(7), + white(12), + delimiter(13), + white(14), + string(15) + ] + }, + { + line: `fruit . flavor = "banana"`, + // ----0123456789 11 14 + tokens: [ + variable(0), + white(5), + delimiter(6), + white(7), + variable(8), + white(14), + delimiter(15), + white(16), + string(17) + ] + }, + { + line: 'fruit.apple.smooth = true', + // ----0123456789 11 14 17 20 + tokens: [ + variable(0), + delimiter(5), + variable(6), + delimiter(11), + variable(12), + white(18), + delimiter(19), + white(20), + boolean(21) + ] + }, + { + line: `3.14159 = "pi"`, + // ----0123456789 11 + tokens: [variable(0), delimiter(1), variable(2), white(7), delimiter(8), white(9), string(10)] + } + ], + // https://toml.io/en/v1.0.0#string + [ + { + line: `str = "I'm a string. \\"You can quote me\\". Name\\tJos\\u00E9\\nLocation\\tSF."`, + // ----0123456789 11 14 17 20 23 26 29 32 35 38 41 44 47 50 53 56 59 62 65 68 71 + tokens: [ + variable(0), + white(3), + delimiter(4), + white(5), + string(6), + escape(21), + string(23), + escape(39), + string(41), + escape(47), + string(49), + escape(52), + // escape(58), this is connected + string(60), + escape(68), + string(70) + ] + }, + { + line: `"\\b \\t \\f \\r \\\\ \\u1234 \\U00012345"`, + // ----01 234 567 8910111314 1617 20 2324 27 30 33 + tokens: [ + variableQuoted(0), + escape(1), + variableQuoted(3), + escape(4), + variableQuoted(6), + escape(7), + variableQuoted(9), + escape(10), + variableQuoted(12), + escape(13), + variableQuoted(15), + escape(16), + variableQuoted(22), + escape(23), + variableQuoted(33) + ] + } + ], + // unterminated strings + [ + { + line: `str = "unterminated`, + tokens: [ + variable(0), + white(3), + delimiter(4), + white(5), + { + startIndex: 6, + type: 'string.invalid.toml' + } + ] + }, + { + line: `str = 'unterminated`, + tokens: [ + variable(0), + white(3), + delimiter(4), + white(5), + { + startIndex: 6, + type: 'string.literal.invalid.toml' + } + ] + }, + { + line: `str = 'untermi\\"nated`, + tokens: [ + variable(0), + white(3), + delimiter(4), + white(5), + { + startIndex: 6, + type: 'string.literal.invalid.toml' + } + ] + }, + { + line: `str = "untermi\\"nated`, + tokens: [ + variable(0), + white(3), + delimiter(4), + white(5), + { + startIndex: 6, + type: 'string.invalid.toml' + } + ] + } + ], + // multiline basic strings + [ + { + line: `str1 = """`, + tokens: [variable(0), white(4), delimiter(5), white(6), stringMulti(7)] + }, + { + line: 'Roses are red', + tokens: [stringMulti(0)] + }, + { + line: `Violets are blue"""`, + tokens: [stringMulti(0)] + }, + { + line: `str3 = """`, + tokens: [variable(0), white(4), delimiter(5), white(6), stringMulti(7)] + }, + { + line: `The quick brown \\`, + // ----0123456789 11 14 + tokens: [stringMulti(0), escape(16)] + }, + { + line: `fox jumps over \\`, + // ----0123456789 11 14 + tokens: [stringMulti(0), escape(15)] + }, + { + line: `the lazy dog."""`, + tokens: [stringMulti(0)] + } + ], + // quotes and escapes in multiline basic strings + [ + { + line: `str4 = """Here are two quotation marks: "". Simple enough."""`, + // ----0123456789 + tokens: [variable(0), white(4), delimiter(5), white(6), stringMulti(7)] + }, + { + line: `str5 = """Here are three quotation marks: ""\\"."""`, + // ----0123456789 11 14 17 20 23 26 29 32 35 38 41 44 47 + tokens: [ + variable(0), + white(4), + delimiter(5), + white(6), + stringMulti(7), + escape(44), + stringMulti(46) + ] + }, + { + line: `str6 = """Here are fifteen quotation marks: ""\\"""\\"""\\"""\\"""\\"."""`, + // ----0123456789 11 14 17 20 23 26 29 32 35 38 41 44 47 50 5254 5658 6062 64 + tokens: [ + variable(0), + white(4), + delimiter(5), + white(6), + stringMulti(7), + escape(46), + stringMulti(48), + escape(50), + stringMulti(52), + escape(54), + stringMulti(56), + escape(58), + stringMulti(60), + escape(62), + stringMulti(64) + ] + }, + { + line: `str7 = """"This," she said, "is just a pointless statement.""""`, + tokens: [variable(0), white(4), delimiter(5), white(6), stringMulti(7)] + } + ], + // literal strings + [ + { + line: `winpath = 'C:\\Users\\nodejs\\templates'`, + tokens: [variable(0), white(7), delimiter(8), white(9), stringLiteral(10)] + }, + { + line: `winpath2 = '\\ServerX\\admin$\\system32\\'`, + tokens: [variable(0), white(8), delimiter(9), white(10), stringLiteral(11)] + }, + { + line: `quoted = 'Tom "Dubs" Preston-Werner'`, + tokens: [variable(0), white(6), delimiter(9), white(10), stringLiteral(11)] + }, + { + line: `regex = '<\\i\\c*\\s*>'`, + tokens: [variable(0), white(5), delimiter(6), white(7), stringLiteral(8)] + } + ], + // multiline literal strings + [ + { + line: `regex2 = '''I [dw]on't need \\d{2} apples'''`, + tokens: [variable(0), white(6), delimiter(7), white(8), stringLiteralMulti(9)] + }, + { + line: `lines = '''`, + tokens: [variable(0), white(5), delimiter(7), white(8), stringLiteralMulti(9)] + }, + { + line: `The first newline is`, + tokens: [stringLiteralMulti(0)] + }, + { + line: `trimmed in raw strings.`, + tokens: [stringLiteralMulti(0)] + }, + { + line: ` All other whitespace`, + tokens: [stringLiteralMulti(0)] + }, + { + line: ` is preserved.'''`, + tokens: [stringLiteralMulti(0)] + }, + + { + line: `quot15 = '''Here are fifteen quotation marks: """""""""""""""'''`, + tokens: [variable(0), white(6), delimiter(7), white(8), stringLiteralMulti(9)] + }, + // sequences of three or more single quotes are not permitted INSIDE the literal string + // (so '''' = ok, ''''' = ok, '''''' = not ok) + { + line: `apos15 = '''Here are fifteen apostrophes: '''''''''''''''''' # INVALID`, + // ----0123456789 11 14 17 20 23 26 29 32 35 38 41 44 47 50 53 56 59 61 + tokens: [ + variable(0), + white(6), + delimiter(7), + white(8), + stringLiteralMulti(9), + // after ''''', rest pairs are interpreted as variable name + variableLiteral(47), + // unterminated variable literal + { + startIndex: 59, + type: 'variable.invalid.toml' + } + // and there's no comment + ] + }, + { + line: `apos15 = "Here are fifteen apostrophes: '''''''''''''''"`, + tokens: [variable(0), white(6), delimiter(7), white(8), string(9)] + }, + { + line: `str = ''''That,' she said, 'is still pointless.''''`, + tokens: [variable(0), white(3), delimiter(4), white(5), stringLiteralMulti(6)] + } + ], + // https://toml.io/en/v1.0.0#integer + [ + testNumber('int1 = +99'), + testNumber('int2 = 42'), + testNumber('int3 = 0'), + testNumber('int4 = -17'), + testNumber('int5 = 1_000'), + testNumber('int6 = 5_349_221'), + testNumber('int7 = 53_49_221'), + testNumber('int8 = 1_2_3_4_5'), + { + line: 'bad = 01234', // leading zero not allowed + tokens: [ + variable(0), + white(3), + delimiter(5), + white(6), + // 01234 becomes next key + variable(7) + ] + }, + testNumber('int9 = +0'), + testNumber('intA = -0') + ], + // hex, octal, binary + [ + testNumber('hex1 = 0xDEADBEEF', hex), + testNumber('hex2 = 0xdeadbeef', hex), + testNumber('hex3 = 0xdead_beef', hex), + testNumber('oct1 = 0o01234567', octal), + testNumber('oct2 = 0o755', octal), + testNumber('bin1 = 0b11010110', binary), + testNumber('bin2 = 0b1101_0110', binary) + ], + // https://toml.io/en/v1.0.0#float + [ + testNumber('flt1 = +1.0', float), + testNumber('flt2 = 3.1415', float), + testNumber('flt3 = -0.01', float), + testNumber('flt4 = 5e+22', float), + testNumber('flt5 = 1e06', float), + testNumber('flt6 = -2E-2', float), + testNumber('flt7 = 6.626e-34', float), + { + line: 'invalid_float_1 = .7', + tokens: [ + variable(0), + white(15), + delimiter(16), + white(17), + { + startIndex: 18, + type: 'source.toml' + }, + variable(19) + ] + }, + { + line: 'invalid_float_2 = 1.', + tokens: [variable(0), white(15), delimiter(16), white(17), variable(18), delimiter(19)] + }, + { + line: 'invalid_float_3 = 6.e+20', + tokens: [ + variable(0), + white(15), + delimiter(16), + white(17), + variable(18), + delimiter(19), + variable(20), + // + is not valid in this context + { + startIndex: 21, + type: 'source.toml' + }, + // 20 can be variable again + variable(22) + ] + }, + testNumber('flt8 = 224_617.445_991_228', float), + testNumber('flt9 = -0.0', float), + testNumber('fltA = +0.0', float), + testNumber('inf1 = inf', infinity), + testNumber('inf2 = +inf', infinity), + testNumber('inf3 = -inf', infinity), + testNumber('nan1 = nan', nan), + testNumber('nan2 = +nan', nan), + testNumber('nan3 = -nan', nan) + ], + // https://toml.io/en/v1.0.0#boolean + [ + { + line: 'bool1 = true', + tokens: [variable(0), white(5), delimiter(6), white(7), boolean(8)] + }, + { + line: 'bool2 = false', + tokens: [variable(0), white(5), delimiter(6), white(7), boolean(8)] + } + ], + // https://toml.io/en/v1.0.0#offset-date-time + // https://toml.io/en/v1.0.0#local-date-time + // https://toml.io/en/v1.0.0#local-date + [ + testNumber('odt1 = 1979-05-27T07:32:00Z', datetime), + testNumber('odt2 = 1979-05-27T00:32:00-07:00', datetime), + testNumber('odt3 = 1979-05-27T00:32:00.999999-07:00', datetime), + testNumber('odt2 = 1979-05-27T00:32:00.1+07:00', datetime), + testNumber('ldt1 = 1979-05-27T07:32:00', datetime), + testNumber('ldt2 = 1979-05-27T00:32:00.999999', datetime), + testNumber('ld_1 = 1979-05-27', date), + testNumber('lt_1 = 23:32:00', time), + testNumber('lt_2 = 00:32:00.999999', time) + ], + // https://toml.io/en/v1.0.0#array + [ + { + line: 'integers = [1, 2, 3]', + // ----0123456789 11 14 17 + tokens: [ + variable(0), + white(8), + delimiter(9), + white(10), + square(11), + decimal(12), + delimiter(13), + white(14), + decimal(15), + delimiter(16), + white(17), + decimal(18), + square(19) + ] + }, + { + line: `colors = ["red", "yellow", "green",]`, + // ----0123456789 11 14 17 20 23 26 29 32 + tokens: [ + variable(0), + white(6), + delimiter(7), + white(8), + square(9), + string(10), + delimiter(15), + white(16), + string(17), + delimiter(25), + white(26), + string(27), + delimiter(34), + square(35) + ] + }, + { + line: `nested_arrays_of_ints = [ [ 1, 2 ], [3, 4, 5] ]`, + // ----0123456789 11 14 17 20 23 26 29 32 35 38 41 44 + tokens: [ + variable(0), + white(21), + delimiter(22), + white(23), + square(24), + white(25), + square(26), + white(27), + decimal(28), + delimiter(29), + white(30), + decimal(31), + white(32), + square(33), + delimiter(34), + white(35), + square(36), + decimal(37), + delimiter(38), + white(39), + decimal(40), + delimiter(41), + white(42), + decimal(43), + square(44), + white(45), + square(46) + ] + }, + { + line: `nested_mixed_array = [ [ 1, 2 ], ["a", "b", "c"] ]`, + // ----0123456789 11 14 17 20 23 26 29 32 35 38 41 44 47 + tokens: [ + variable(0), + white(18), + delimiter(19), + white(20), + square(21), + white(22), + square(23), + white(24), + decimal(25), + delimiter(26), + white(27), + decimal(28), + white(29), + square(30), + delimiter(31), + white(32), + square(33), + string(34), + delimiter(37), + white(38), + string(39), + delimiter(42), + white(43), + string(44), + square(47), + white(48), + square(49) + ] + }, + { + line: `string_array = [ "all", 'strings', """are the same""", '''type''' ]`, + // ----0123456789 11 14 17 20 23 26 29 32 35 38 41 44 47 50 53 56 59 62 65 + tokens: [ + variable(0), + white(12), + delimiter(13), + white(14), + square(15), + white(16), + string(17), + delimiter(22), + white(23), + stringLiteral(24), + delimiter(33), + white(34), + stringMulti(35), + delimiter(53), + white(54), + stringLiteralMulti(55), + white(65), + square(66) + ] + } + ], + // Multiline arrays + [ + { + line: 'integers2 = [', + // ----0123456789 + tokens: [variable(0), white(9), delimiter(10), white(11), square(12)] + }, + { + line: ' 1, 2, 3', + // ----012345678 + tokens: [ + white(0), + decimal(2), + delimiter(3), + white(4), + decimal(5), + delimiter(6), + white(7), + decimal(8) + ] + }, + { + line: ']', + tokens: [square(0)] + }, + + { + line: 'integers3 = [', + // ----0123456789 + tokens: [variable(0), white(9), delimiter(10), white(11), square(12)] + }, + { + line: ' 1,', + tokens: [white(0), decimal(2), delimiter(3)] + }, + { + line: ' 2, # this is ok', + tokens: [white(0), decimal(2), delimiter(3), white(4), comment(5)] + }, + { + line: ']', + tokens: [square(0)] + } + ], + // https://toml.io/en/v1.0.0#table + [ + { + line: '[table]', + tokens: [square(0), table(1), square(6)] + }, + { + line: '[table-1]', + tokens: [square(0), table(1), square(8)] + }, + { + line: `[dog."tater.man"]`, + // ----0123456789 11 14 + tokens: [square(0), table(1), delimiter(4), tableString(5), square(16)] + }, + { + line: `[a.b.c]`, + // ----0123456 + tokens: [square(0), table(1), delimiter(2), table(3), delimiter(4), table(5), square(6)] + }, + { + line: `[ d.'e-e'.f ]`, + // ----0123456789 11 + tokens: [ + square(0), + white(1), + table(2), + delimiter(3), + tableLiteral(4), + delimiter(9), + table(10), + white(11), + square(12) + ] + }, + { + line: `[ g . h . i ]`, + // ----0123456789 11 14 + tokens: [ + square(0), + white(1), + table(2), + white(3), + delimiter(4), + white(5), + table(7), + white(8), + delimiter(10), + white(11), + table(12), + white(13), + square(14) + ] + }, + { + line: `[ j . "ʞ" . 'l' ]`, + // ----0123456789 11 14 + tokens: [ + square(0), + white(1), + table(2), + white(3), + delimiter(4), + white(5), + tableString(6), + white(9), + delimiter(10), + white(11), + tableLiteral(12), + white(15), + square(16) + ] + }, + { + line: ' [indented]', + // ----0123456789 11 + tokens: [white(0), square(4), table(5), square(13)] + } + ], + // https://toml.io/en/v1.0.0#inline-table + [ + { + line: `name = { first = "Tom", last = "Preston-Werner" }`, + // ----0123456789 11 14 17 20 23 26 29 32 35 38 41 44 47 + tokens: [ + variable(0), + white(4), + delimiter(5), + white(6), + bracket(7), + white(8), + variable(9), + white(14), + delimiter(15), + white(16), + string(17), + delimiter(22), + white(23), + variable(24), + white(28), + delimiter(29), + white(30), + string(31), + white(47), + bracket(48) + ] + }, + { + line: 'point = { x = 1, y = 2 }', + // ----0123456789 11 14 17 20 23 + tokens: [ + variable(0), + white(5), + delimiter(6), + white(7), + bracket(8), + white(9), + variable(10), + white(11), + delimiter(12), + white(13), + decimal(14), + delimiter(15), + white(16), + variable(17), + white(18), + delimiter(19), + white(20), + decimal(21), + white(22), + bracket(23) + ] + }, + { + line: `animal = { type.name = "pug" }`, + // ----0123456789 11 14 17 20 23 26 29 + tokens: [ + variable(0), + white(6), + delimiter(7), + white(8), + bracket(9), + white(10), + variable(11), + delimiter(15), + variable(16), + white(20), + delimiter(21), + white(22), + string(23), + white(28), + bracket(29) + ] + } + ], + // multi line/nested array test case + [ + { + line: `shop = {`, + tokens: [variable(0), white(4), delimiter(5), white(6), bracket(7)] + }, + { + line: ` products = [`, + // ----0123456789 11 + tokens: [white(0), variable(2), white(10), delimiter(11), white(12), square(13)] + }, + { + line: ` { name = "Hammer", sku = 738594937 },`, + // ----0123456789 11 14 17 20 23 26 29 32 35 38 + tokens: [ + white(0), + bracket(4), + white(5), + variable(6), + white(10), + delimiter(11), + white(12), + string(13), + delimiter(21), + white(22), + variable(23), + white(26), + delimiter(27), + white(28), + decimal(29), + white(38), + bracket(39), + delimiter(40) + ] + }, + { + line: ` {},`, + // ----0123456 + tokens: [ + white(0), + bracket(4), + // bracket(5), // connected + delimiter(6) + ] + }, + { + line: ` { name = "Nail", sku = 284758393, color = "gray" }`, + // ----0123456789 11 14 17 20 23 26 29 32 35 38 41 44 47 50 53 + tokens: [ + white(0), + bracket(4), + white(5), + variable(6), + white(10), + delimiter(11), + white(12), + string(13), + delimiter(19), + white(20), + variable(21), + white(24), + delimiter(25), + white(26), + decimal(27), + delimiter(36), + white(37), + variable(38), + white(43), + delimiter(44), + white(45), + string(46), + white(52), + bracket(53) + ] + }, + { + line: `]}`, + tokens: [square(0), bracket(1)] + } + ], + // https://toml.io/en/v1.0.0#array-of-tables + [ + { + line: `[[products]]`, + // ----0123456789 11 + tokens: [square(0), table(2), square(10)] + }, + { + line: `[[fruits.varieties]]`, + // ----0123456789 11 14 17 + tokens: [square(0), table(2), delimiter(8), table(9), square(18)] + } + ] +]); diff --git a/src/basic-languages/toml/toml.ts b/src/basic-languages/toml/toml.ts new file mode 100644 index 0000000000..ffdd050d59 --- /dev/null +++ b/src/basic-languages/toml/toml.ts @@ -0,0 +1,410 @@ +import { languages } from '../../fillers/monaco-editor-core'; + +export const conf: languages.LanguageConfiguration = { + comments: { + lineComment: '#' + }, + brackets: [ + ['{', '}'], + ['[', ']'], + ['(', ')'] + ], + autoClosingPairs: [ + { open: '{', close: '}' }, + { open: '[', close: ']' }, + { open: '(', close: ')' }, + { open: '"', close: '"' }, + { open: "'", close: "'" } + ], + folding: { + offSide: true + }, + onEnterRules: [ + { + beforeText: /[\{\[]\s*$/, + action: { + indentAction: languages.IndentAction.Indent + } + } + ] +}; +export const language: languages.IMonarchLanguage = { + tokenPostfix: '.toml', + brackets: [ + { token: 'delimiter.bracket', open: '{', close: '}' }, + { token: 'delimiter.square', open: '[', close: ']' } + ], + + // https://toml.io/en/v1.0.0#integer + // 0, +0, -0 are valid, otherwise leading 0 is not allowed + // _ needs to have at least one digit on each side + numberInteger: /[+-]?(0|[1-9](_?[0-9])*)/, + numberOctal: /0o[0-7](_?[0-7])*/, + numberHex: /0x[0-9a-fA-F](_?[0-9a-fA-F])*/, + numberBinary: /0b[01](_?[01])*/, + + floatFractionPart: /\.[0-9](_?[0-9])*/, + // exponent can include leading zeros + floatExponentPart: /[eE][+-]?[0-9](_?[0-9])*/, + + // RFC 3339 data times + date: /\d{4}-\d\d-\d\d/, + time: /\d\d:\d\d:\d\d(\.\d+)?/, + offset: /[+-]\d\d:\d\d/, + + // https://toml.io/en/v1.0.0#string + escapes: /\\([btnfr"\\]|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8})/, + identifier: /([\w-]+)/, + // Characters that can start an identifier chain for key or table name + identChainStart: /([\w-"'])/, + // Characters that can start a value + // - '"' and ' for strings + // - t/f for booleans + // - +-, 0-9 for numbers + // - i/n for special numbers + // - [ for arrays, { for tables + valueStart: /(["'tf0-9+\-in\[\{])/, + + tokenizer: { + root: [ + { include: '@comment' }, + { include: '@whitespace' }, + // key value pair + [/@identChainStart/, '@rematch', '@kvpair'], + + // table + [/\[/, '@brackets', '@table'], + + // *invalid* value without a key, still parse + // the value so it doesn't become a key and mess up + // further parsing + [/=/, 'delimiter', '@value'] + ], + comment: [[/#.*$/, 'comment']], + whitespace: [[/[ \t\r\n]+/, 'white']], + + // Parsing a key value pair + kvpair: [ + { include: '@whitespace' }, + { include: '@comment' }, + [/@identChainStart/, '@rematch', '@identChain.variable'], + // switch to value, so we pop back to root when + // it's done + [ + /=/, + { + token: 'delimiter', + switchTo: '@value' + } + ], + [/./, '@rematch', '@pop'] + ], + + // Parsing a key identifier + ...createIdentChainStates('variable'), + + // Parsing a top level [table] + table: [ + { include: '@whitespace' }, + { include: '@comment' }, + // increase nesting + [/\[/, '@brackets', '@table'], + [/@identChainStart/, '@rematch', '@identChain.type'], + [/\]/, '@brackets', '@pop'] + ], + + // Table name identifier + ...createIdentChainStates('type'), + + // A top level value (in a kvpair) + value: [ + { include: '@whitespace' }, + { include: '@comment' }, + { include: '@value.cases' }, + // not valid value + [/./, '@rematch', '@pop'] + ], + + 'value.string.singleQuoted': createSingleLineLiteralStringState('string.literal'), + 'value.string.doubleQuoted': createSingleLineStringState('string'), + 'value.string.multi.doubleQuoted': [ + // anything not a quote or \ (escape char) is part of the string + [/[^"\\]+/, 'string.multi'], + + // for more compatibility with themes, escape token classes are the same everywhere + [/@escapes/, 'constant.character.escape'], + // end of line continuation + [/\\$/, `constant.character.escape`], + // invalid escape sequence + [/\\./, `constant.character.escape.invalid`], + + // the spec doesn't explicitly mention 3 or more quotes + // are invalid, but it mentions 1 or 2 quotes are valid inside + // multiline, so here we assume the rule is the same as literal multiline + [/"""(""|")?/, 'string.multi', '@pop'], + + // not terminated by single " + [/"/, 'string.multi'] + ], + 'value.string.multi.singleQuoted': [ + // anything not ' is part of the string + [/[^']+/, 'string.literal.multi'], + // 3-5 ' ends the string + [/'''(''|')?/, 'string.literal.multi', '@pop'], + + // not terminated by single ' + [/'/, 'string.literal.multi'] + ], + + // Arrays + 'value.array': [ + { include: '@whitespace' }, + { include: '@comment' }, + // closing the array + [/\]/, '@brackets', '@pop'], + // seprator + [/,/, 'delimiter'], + // values in the array + [/@valueStart/, '@rematch', '@value.array.entry'], + + // invalid syntax, skip until , or ] + [/.+(?=[,\]])/, 'source'] + ], + + // One entry in the array + 'value.array.entry': [ + { include: '@whitespace' }, + { include: '@comment' }, + // values in the array - pops if matches + { include: '@value.cases' }, + // invalid syntax, skip until , or ] + [/.+(?=[,\]])/, 'source', '@pop'], + // unterminated array, just give up + // and skip one character + [/./, 'source', '@pop'] + ], + + // Inline-tables + 'value.inlinetable': [ + { include: '@whitespace' }, + { include: '@comment' }, + // closing the table + [/\}/, '@brackets', '@pop'], + // seprator + [/,/, 'delimiter'], + // key-value pairs in the table + [/@identChainStart/, '@rematch', '@value.inlinetable.entry'], + + // *invalid* value without a key, still parse + // the value so it doesn't become a key and mess up + // further parsing + [/=/, 'delimiter', '@value.inlinetable.value'], + + // *invalid* value without key or = + [/@valueStart/, '@rematch', '@value.inlinetable.value'], + // invalid syntax, skip until , or } + [/.+(?=[,\}])/, 'source', '@pop'] + ], + + // One entry (key-value pair) in the inline table + 'value.inlinetable.entry': [ + { include: '@whitespace' }, + { include: '@comment' }, + + // key + [/@identChainStart/, '@rematch', '@identChain.variable'], + // = value + [ + /=/, + { + token: 'delimiter', + switchTo: '@value.inlinetable.value' + } + ], + // invalid syntax, skip until , or } + [/.+(?=[,\}])/, 'source', '@pop'] + ], + + // One value entry in the inline table + 'value.inlinetable.value': [ + { include: '@whitespace' }, + { include: '@comment' }, + // values in the table - pops back to inlinetable if matches + { include: '@value.cases' }, + // invalid syntax, skip until , or } + [/.+(?=[,\}])/, 'source', '@pop'], + // unterminated table, just give up + // and skip one character + [/./, 'source', '@pop'] + ], + + 'value.cases': [ + // basic (double quote) strings + [ + /"""/, + { + token: 'string.multi', + switchTo: '@value.string.multi.doubleQuoted' + } + ], + [/"(\\.|[^"])*$/, 'string.invalid'], // unterminated + [ + /"/, + { + token: 'string', + switchTo: '@value.string.doubleQuoted' + } + ], + + // literal (single quote) strings + [ + /'''/, + { + token: 'string.literal.multi', + switchTo: '@value.string.multi.singleQuoted' + } + ], + [/'[^']*$/, 'string.literal.invalid'], // unterminated + [ + /'/, + { + token: 'string.literal', + switchTo: '@value.string.singleQuoted' + } + ], + + // boolean + [/(true|false)/, 'constant.language.boolean', '@pop'], + + // arrays + [ + /\[/, + { + token: '@brackets', + switchTo: '@value.array' + } + ], + + // inline tables + [ + /\{/, + { + token: '@brackets', + switchTo: '@value.inlinetable' + } + ], + + // integer type + // require integer to be not followed by invalid tokens, + // so it can run before the other types (since it's more common), + // and not clash with other types + // - 0-9 for integers with leading 0 + // - _ for separators (otherwise 123_456.789 would accept 123) + // - oxb for hex, octal, binary + // - \. and eE for floats + // - '-' and ':' for date and time + [/@numberInteger(?![0-9_oxbeE\.:-])/, 'number', '@pop'], + + // float + [ + /@numberInteger(@floatFractionPart@floatExponentPart?|@floatExponentPart)/, + 'number.float', + '@pop' + ], + + // integer types + [/@numberOctal/, 'number.octal', '@pop'], + [/@numberHex/, 'number.hex', '@pop'], + [/@numberBinary/, 'number.binary', '@pop'], + + // special float + [/[+-]?inf/, 'number.inf', '@pop'], + [/[+-]?nan/, 'number.nan', '@pop'], + + // Date Time (offset and local) + [/@date[Tt ]@time(@offset|Z)?/, 'number.datetime', '@pop'], + [/@date/, 'number.date', '@pop'], + [/@time/, 'number.time', '@pop'] + ] + } +}; + +type State = languages.IMonarchLanguageRule[]; + +/** + * Create states for parsing a TOML identifier chain, + * like `key`, `key.subkey`, `key."foo.bar"`, etc., + * using the given token class. + */ +function createIdentChainStates(tokenClass: string): Record { + const singleQuotedState = `identChain.${tokenClass}.singleQuoted`; + const singleQuoteClass = `${tokenClass}.string.literal`; + const doubleQuotedState = `identChain.${tokenClass}.doubleQuoted`; + const doubleQuoteClass = `${tokenClass}.string`; + return { + [`identChain.${tokenClass}`]: [ + { include: '@whitespace' }, + { include: '@comment' }, + + [/@identifier/, tokenClass], + [/\./, 'delimiter'], + + // string literal + [/'[^']*$/, `${tokenClass}.invalid`], // unterminated + [ + /'/, + { + token: singleQuoteClass, + next: `@${singleQuotedState}` + } + ], + + // string + [/"(\\.|[^"])*$/, `${tokenClass}.invalid`], // unterminated + [ + /"/, + { + token: doubleQuoteClass, + next: `@${doubleQuotedState}` + } + ], + + // end of identifier chain + [/./, '@rematch', '@pop'] + ], + [singleQuotedState]: createSingleLineLiteralStringState(singleQuoteClass), + [doubleQuotedState]: createSingleLineStringState(doubleQuoteClass) + }; +} + +/** + * Create a state for parsing a single line literal string + * (i.e a 'single quoted string') using the given token class. + */ +function createSingleLineLiteralStringState(tokenClass: string): State { + return [ + // anything not a single quote + [/[^']+/, tokenClass], + // end of string + [/'/, tokenClass, '@pop'] + ]; +} + +/** + * Create a state for parsing a single line string + * (i.e a "double quoted string") using the given token class. + */ +function createSingleLineStringState(tokenClass: string): State { + return [ + // anything not a quote or \ (escape char) is part of the string + [/[^"\\]+/, tokenClass], + + // for more compatibility with themes, escape token classes are the same everywhere + [/@escapes/, 'constant.character.escape'], + // invalid escape sequence + [/\\./, `constant.character.escape.invalid`], + + // end of string + [/"/, tokenClass, '@pop'] + ]; +} diff --git a/website/src/website/data/home-samples/sample.toml.txt b/website/src/website/data/home-samples/sample.toml.txt new file mode 100644 index 0000000000..c071a68de0 --- /dev/null +++ b/website/src/website/data/home-samples/sample.toml.txt @@ -0,0 +1,151 @@ +# This example is taken from https://github.com/rust-lang/rust +# and is licensed under MIT/Apache-2.0 +# The content is modified so it's not too big + +# This file's format is documented at +# https://forge.rust-lang.org/triagebot/pr-assignment.html#configuration + +[relabel] +allow-unauthenticated = [ + "A-*", + "C-*", + "D-*", + "E-*", + "F-*", + "I-*", + "L-*", + "has-merge-commits", +] + +[review-submitted] +# This label is added when a "request changes" review is submitted. +reviewed_label = "S-waiting-on-author" +# These labels are removed when a "request changes" review is submitted. +review_labels = ["S-waiting-on-review"] + +[glacier] + +[ping.icebreakers-llvm] +alias = ["llvm", "llvms"] +message = """\ +Hey LLVM ICE-breakers! This bug has been identified as a good +"LLVM ICE-breaking candidate". In case it's useful, here are some +[instructions] for tackling these sorts of bugs. Maybe take a look? +Thanks! <3 + +[instructions]: https://rustc-dev-guide.rust-lang.org/notification-groups/llvm.html +""" +label = "ICEBreaker-LLVM" + +[ping.windows] +message = """\ +Hey Windows Group! This bug has been identified as a good "Windows candidate". +In case it's useful, here are some [instructions] for tackling these sorts of +bugs. Maybe take a look? +Thanks! <3 + +[instructions]: https://rustc-dev-guide.rust-lang.org/notification-groups/windows.html +""" +label = "O-windows" + +[prioritize] +label = "I-prioritize" + +[autolabel."I-prioritize"] +trigger_labels = [ + "regression-untriaged", + "regression-from-stable-to-stable", + "regression-from-stable-to-beta", + "regression-from-stable-to-nightly", + "I-unsound", +] +exclude_labels = [ + "P-*", + "T-infra", + "T-release", + "requires-nightly", +] + +[autolabel."needs-triage"] +new_issue = true +exclude_labels = [ + "C-tracking-issue", + "A-diagnostics", +] + +[notify-zulip."I-prioritize"] +zulip_stream = 245100 # #t-compiler/wg-prioritization/alerts +topic = "#{number} {title}" +message_on_add = """\ +@*WG-prioritization/alerts* issue #{number} has been requested for prioritization. + +# [Procedure](https://forge.rust-lang.org/compiler/prioritization/procedure.html#assign-priority-to-unprioritized-issues-with-i-prioritize-label) +- Priority? +- Regression? +- Notify people/groups? +- Needs `I-{team}-nominated`? +""" +message_on_remove = "Issue #{number}'s prioritization request has been removed." +message_on_close = "Issue #{number} has been closed while requested for prioritization." +message_on_reopen = "Issue #{number} has been reopened." + +# FIXME: Patch triagebot to support `notify-zulip.