From ca090998ba51f042c463242b1bcfeda4b23138fa Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 26 Oct 2023 18:00:41 -0700 Subject: [PATCH 01/24] Initial scanner and parser --- packages/compiler/src/core/parser.ts | 95 +++++++++++++++++++ packages/compiler/src/core/scanner.ts | 59 +++++++++++- packages/compiler/src/core/types.ts | 40 ++++++++ .../compiler/src/formatter/print/printer.ts | 5 + 4 files changed, 197 insertions(+), 2 deletions(-) diff --git a/packages/compiler/src/core/parser.ts b/packages/compiler/src/core/parser.ts index d3c66f9c4a..0bbcb7386d 100644 --- a/packages/compiler/src/core/parser.ts +++ b/packages/compiler/src/core/parser.ts @@ -90,6 +90,11 @@ import { SourceFile, Statement, StringLiteralNode, + StringTemplateExpressionNode, + StringTemplateHeadNode, + StringTemplateMiddleNode, + StringTemplateSpanNode, + StringTemplateTailNode, Sym, SyntaxKind, TemplateParameterDeclarationNode, @@ -1340,6 +1345,8 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa return parseReferenceExpression(); case Token.StringLiteral: return parseStringLiteral(); + case Token.StringTemplateHead: + return parseStringTemplateExpression(); case Token.TrueKeyword: case Token.FalseKeyword: return parseBooleanLiteral(); @@ -1446,6 +1453,82 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa }; } + function parseStringTemplateExpression(): StringTemplateExpressionNode { + const pos = tokenPos(); + const head = parseStringTemplateHead(); + const spans = parseStringTemplateSpans(); + return { + kind: SyntaxKind.StringTemplateExpression, + head, + spans, + ...finishNode(pos), + }; + } + + function parseStringTemplateHead(): StringTemplateHeadNode { + const pos = tokenPos(); + const text = tokenValue(); + parseExpected(Token.StringTemplateHead); + + return { + kind: SyntaxKind.StringTemplateHead, + text, + ...finishNode(pos), + }; + } + + function parseStringTemplateSpans(): readonly StringTemplateSpanNode[] { + const list: StringTemplateSpanNode[] = []; + let node: StringTemplateSpanNode; + do { + node = parseTemplateTypeSpan(); + list.push(node); + } while (node.literal.kind === SyntaxKind.StringTemplateMiddle); + return list; + } + + function parseTemplateTypeSpan(): StringTemplateSpanNode { + const pos = tokenPos(); + const expression = parseExpression(); + const literal = parseLiteralOfTemplateSpan(); + return { + kind: SyntaxKind.StringTemplateSpan, + literal, + expression, + ...finishNode(pos), + }; + } + function parseLiteralOfTemplateSpan(): StringTemplateMiddleNode | StringTemplateTailNode { + const pos = tokenPos(); + if (token() === Token.CloseBrace) { + nextStringTemplateToken(); + return parseTemplateMiddleOrTemplateTail(); + } else { + parseExpected(Token.StringTemplateTail); + return { + kind: SyntaxKind.StringTemplateTail, + text: "", + ...finishNode(pos), + }; + } + } + + function parseTemplateMiddleOrTemplateTail(): StringTemplateMiddleNode | StringTemplateTailNode { + const pos = tokenPos(); + const text = tokenValue(); + const kind = + token() === Token.StringTemplateMiddle + ? SyntaxKind.StringTemplateMiddle + : SyntaxKind.StringTemplateTail; + + nextToken(); + return { + kind, + text, + ...finishNode(pos), + }; + } + function parseNumericLiteral(): NumericLiteralNode { const pos = tokenPos(); const valueAsString = tokenValue(); @@ -2575,6 +2658,10 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa scanner.scanDoc(); } + function nextStringTemplateToken() { + scanner.scanStringTemplate(); + } + function createMissingIdentifier(): IdentifierNode { const pos = tokenPos(); previousTokenEnd = pos; @@ -3162,7 +3249,15 @@ export function visitChildren(node: Node, cb: NodeCallback): T | undefined case SyntaxKind.DocUnknownTag: return visitNode(cb, node.tagName) || visitEach(cb, node.content); + case SyntaxKind.StringTemplateExpression: + return visitNode(cb, node.head) || visitEach(cb, node.spans); + case SyntaxKind.StringTemplateSpan: + return visitNode(cb, node.expression) || visitNode(cb, node.literal); + // no children for the rest of these. + case SyntaxKind.StringTemplateHead: + case SyntaxKind.StringTemplateMiddle: + case SyntaxKind.StringTemplateTail: case SyntaxKind.StringLiteral: case SyntaxKind.NumericLiteral: case SyntaxKind.BooleanLiteral: diff --git a/packages/compiler/src/core/scanner.ts b/packages/compiler/src/core/scanner.ts index ae52583a76..618bbb7d20 100644 --- a/packages/compiler/src/core/scanner.ts +++ b/packages/compiler/src/core/scanner.ts @@ -30,6 +30,9 @@ export enum Token { Identifier, NumericLiteral, StringLiteral, + StringTemplateHead, + StringTemplateMiddle, + StringTemplateTail, // Add new tokens above if they don't fit any of the categories below /////////////////////////////////////////////////////////////// @@ -165,6 +168,11 @@ export type DocToken = | Token.DocCodeFenceDelimiter | Token.EndOfFile; +export type StringTemplateToken = + | Token.StringTemplateHead + | Token.StringTemplateMiddle + | Token.StringTemplateTail; + /** @internal */ export const TokenDisplay = getTokenDisplayTable([ [Token.None, "none"], @@ -175,6 +183,9 @@ export const TokenDisplay = getTokenDisplayTable([ [Token.ConflictMarker, "conflict marker"], [Token.NumericLiteral, "numeric literal"], [Token.StringLiteral, "string literal"], + [Token.StringTemplateHead, "string template head"], + [Token.StringTemplateMiddle, "string template middle"], + [Token.StringTemplateTail, "string template tail"], [Token.NewLine, "newline"], [Token.Whitespace, "whitespace"], [Token.DocCodeFenceDelimiter, "doc code fence delimiter"], @@ -298,6 +309,8 @@ export interface Scanner { /** Advance one token inside DocComment. Use inside {@link scanRange} callback over DocComment range. */ scanDoc(): DocToken; + scanStringTemplate(): StringTemplateToken; + /** Reset the scanner to the given start and end positions, invoke the callback, and then restore scanner state. */ scanRange(range: TextRange, callback: () => T): T; @@ -384,6 +397,7 @@ export function createScanner( scan, scanRange, scanDoc, + scanStringTemplate, eof, getTokenText, getTokenValue, @@ -642,6 +656,12 @@ export function createScanner( return (token = Token.EndOfFile); } + function scanStringTemplate(): StringTemplateToken { + tokenPosition = position; + tokenFlags = TokenFlags.None; + return scanStringTemplateSpan(); + } + function scanRange(range: TextRange, callback: () => T): T { const savedPosition = position; const savedEndPosition = endPosition; @@ -820,7 +840,7 @@ export function createScanner( return unterminated(Token.DocCodeSpan); } - function scanString(): Token.StringLiteral { + function scanString(): Token.StringLiteral | Token.StringTemplateHead { position++; // consume '"' loop: for (; !eof(); position++) { @@ -836,6 +856,12 @@ export function createScanner( case CharCode.DoubleQuote: position++; return (token = Token.StringLiteral); + case CharCode.$: + if (lookAhead(1) === CharCode.OpenBrace) { + position += 2; + return (token = Token.StringTemplateHead); + } + continue; case CharCode.CarriageReturn: case CharCode.LineFeed: break loop; @@ -845,7 +871,36 @@ export function createScanner( return unterminated(Token.StringLiteral); } - function scanTripleQuotedString(): Token.StringLiteral { + function scanStringTemplateSpan(): Token.StringTemplateMiddle | Token.StringTemplateTail { + loop: for (; !eof(); position++) { + const ch = input.charCodeAt(position); + switch (ch) { + case CharCode.Backslash: + tokenFlags |= TokenFlags.Escaped; + position++; + if (eof()) { + break loop; + } + continue; + case CharCode.DoubleQuote: + position++; + return (token = Token.StringTemplateTail); + case CharCode.$: + if (lookAhead(1) === CharCode.OpenBrace) { + position += 2; + return (token = Token.StringTemplateMiddle); + } + continue; + case CharCode.CarriageReturn: + case CharCode.LineFeed: + break loop; + } + } + + return unterminated(Token.StringTemplateTail); + } + + function scanTripleQuotedString(): Token.StringLiteral | Token.StringTemplateHead { tokenFlags |= TokenFlags.TripleQuoted; position += 3; // consume '"""' diff --git a/packages/compiler/src/core/types.ts b/packages/compiler/src/core/types.ts index 524b7225c7..b4b9722593 100644 --- a/packages/compiler/src/core/types.ts +++ b/packages/compiler/src/core/types.ts @@ -736,6 +736,11 @@ export enum SyntaxKind { StringLiteral, NumericLiteral, BooleanLiteral, + StringTemplateExpression, + StringTemplateHead, + StringTemplateMiddle, + StringTemplateTail, + StringTemplateSpan, ExternKeyword, VoidKeyword, NeverKeyword, @@ -858,6 +863,10 @@ export type Node = | Statement | Expression | FunctionParameterNode + | StringTemplateSpanNode + | StringTemplateHeadNode + | StringTemplateMiddleNode + | StringTemplateTailNode | Modifier | DocNode | DocContent @@ -1040,6 +1049,7 @@ export type Expression = | StringLiteralNode | NumericLiteralNode | BooleanLiteralNode + | StringTemplateExpressionNode | VoidKeywordNode | NeverKeywordNode | AnyKeywordNode; @@ -1239,6 +1249,36 @@ export interface BooleanLiteralNode extends BaseNode { readonly value: boolean; } +export interface StringTemplateExpressionNode extends BaseNode { + readonly kind: SyntaxKind.StringTemplateExpression; + readonly head: StringTemplateHeadNode; + readonly spans: readonly StringTemplateSpanNode[]; +} + +// Each of these corresponds to a substitution expression and a template literal, in that order. +// The template literal must have kind TemplateMiddleLiteral or TemplateTailLiteral. +export interface StringTemplateSpanNode extends BaseNode { + readonly kind: SyntaxKind.StringTemplateSpan; + readonly expression: Expression; + readonly literal: StringTemplateMiddleNode | StringTemplateTailNode; +} + +export interface StringTemplateLiteralLikeNode extends BaseNode { + text: string; +} + +export interface StringTemplateHeadNode extends StringTemplateLiteralLikeNode { + readonly kind: SyntaxKind.StringTemplateHead; +} + +export interface StringTemplateMiddleNode extends StringTemplateLiteralLikeNode { + readonly kind: SyntaxKind.StringTemplateMiddle; +} + +export interface StringTemplateTailNode extends StringTemplateLiteralLikeNode { + readonly kind: SyntaxKind.StringTemplateTail; +} + export interface ExternKeywordNode extends BaseNode { readonly kind: SyntaxKind.ExternKeyword; } diff --git a/packages/compiler/src/formatter/print/printer.ts b/packages/compiler/src/formatter/print/printer.ts index 6f4cd7b385..85f75c91f7 100644 --- a/packages/compiler/src/formatter/print/printer.ts +++ b/packages/compiler/src/formatter/print/printer.ts @@ -357,6 +357,11 @@ export function printNode( return ""; case SyntaxKind.EmptyStatement: return ""; + case SyntaxKind.StringTemplateExpression: + case SyntaxKind.StringTemplateSpan: + case SyntaxKind.StringTemplateHead: + case SyntaxKind.StringTemplateMiddle: + case SyntaxKind.StringTemplateTail: case SyntaxKind.JsSourceFile: case SyntaxKind.JsNamespaceDeclaration: case SyntaxKind.InvalidStatement: From 3b9ba72b8a02f5f3a99d21c21ffdff709fc05395 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 27 Oct 2023 10:23:58 -0700 Subject: [PATCH 02/24] Progress --- packages/compiler/src/core/parser.ts | 24 ++-- packages/compiler/src/core/scanner.ts | 192 +++++++++++++++----------- packages/compiler/src/core/types.ts | 4 + packages/compiler/test/parser.test.ts | 90 ++++++++++-- 4 files changed, 215 insertions(+), 95 deletions(-) diff --git a/packages/compiler/src/core/parser.ts b/packages/compiler/src/core/parser.ts index 0bbcb7386d..f1b28850ad 100644 --- a/packages/compiler/src/core/parser.ts +++ b/packages/compiler/src/core/parser.ts @@ -1456,7 +1456,7 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa function parseStringTemplateExpression(): StringTemplateExpressionNode { const pos = tokenPos(); const head = parseStringTemplateHead(); - const spans = parseStringTemplateSpans(); + const spans = parseStringTemplateSpans(head.tokenFlags); return { kind: SyntaxKind.StringTemplateExpression, head, @@ -1468,29 +1468,31 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa function parseStringTemplateHead(): StringTemplateHeadNode { const pos = tokenPos(); const text = tokenValue(); + const flags = tokenFlags(); parseExpected(Token.StringTemplateHead); return { kind: SyntaxKind.StringTemplateHead, text, + tokenFlags: flags, ...finishNode(pos), }; } - function parseStringTemplateSpans(): readonly StringTemplateSpanNode[] { + function parseStringTemplateSpans(tokenFlags: TokenFlags): readonly StringTemplateSpanNode[] { const list: StringTemplateSpanNode[] = []; let node: StringTemplateSpanNode; do { - node = parseTemplateTypeSpan(); + node = parseTemplateTypeSpan(tokenFlags); list.push(node); } while (node.literal.kind === SyntaxKind.StringTemplateMiddle); return list; } - function parseTemplateTypeSpan(): StringTemplateSpanNode { + function parseTemplateTypeSpan(tokenFlags: TokenFlags): StringTemplateSpanNode { const pos = tokenPos(); const expression = parseExpression(); - const literal = parseLiteralOfTemplateSpan(); + const literal = parseLiteralOfTemplateSpan(tokenFlags); return { kind: SyntaxKind.StringTemplateSpan, literal, @@ -1498,16 +1500,19 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa ...finishNode(pos), }; } - function parseLiteralOfTemplateSpan(): StringTemplateMiddleNode | StringTemplateTailNode { + function parseLiteralOfTemplateSpan( + headTokenFlags: TokenFlags + ): StringTemplateMiddleNode | StringTemplateTailNode { const pos = tokenPos(); if (token() === Token.CloseBrace) { - nextStringTemplateToken(); + nextStringTemplateToken(headTokenFlags); return parseTemplateMiddleOrTemplateTail(); } else { parseExpected(Token.StringTemplateTail); return { kind: SyntaxKind.StringTemplateTail, text: "", + tokenFlags: tokenFlags(), ...finishNode(pos), }; } @@ -1525,6 +1530,7 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa return { kind, text, + tokenFlags: tokenFlags(), ...finishNode(pos), }; } @@ -2658,8 +2664,8 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa scanner.scanDoc(); } - function nextStringTemplateToken() { - scanner.scanStringTemplate(); + function nextStringTemplateToken(tokenFlags: TokenFlags) { + scanner.reScanStringTemplate(tokenFlags); } function createMissingIdentifier(): IdentifierNode { diff --git a/packages/compiler/src/core/scanner.ts b/packages/compiler/src/core/scanner.ts index 618bbb7d20..2898e6150b 100644 --- a/packages/compiler/src/core/scanner.ts +++ b/packages/compiler/src/core/scanner.ts @@ -309,7 +309,11 @@ export interface Scanner { /** Advance one token inside DocComment. Use inside {@link scanRange} callback over DocComment range. */ scanDoc(): DocToken; - scanStringTemplate(): StringTemplateToken; + /** + * Unconditionally back up and scan a template expression portion. + * @param tokenFlags Token Flags for head StringTemplateToken + */ + reScanStringTemplate(tokenFlags: TokenFlags): StringTemplateToken; /** Reset the scanner to the given start and end positions, invoke the callback, and then restore scanner state. */ scanRange(range: TextRange, callback: () => T): T; @@ -397,7 +401,7 @@ export function createScanner( scan, scanRange, scanDoc, - scanStringTemplate, + reScanStringTemplate, eof, getTokenText, getTokenValue, @@ -414,7 +418,10 @@ export function createScanner( function getTokenValue() { switch (token) { case Token.StringLiteral: - return getStringTokenValue(); + case Token.StringTemplateHead: + case Token.StringTemplateMiddle: + case Token.StringTemplateTail: + return getStringTokenValue(token, tokenFlags); case Token.Identifier: return getIdentifierTokenValue(); default: @@ -563,8 +570,8 @@ export function createScanner( case CharCode.DoubleQuote: return lookAhead(1) === CharCode.DoubleQuote && lookAhead(2) === CharCode.DoubleQuote - ? scanTripleQuotedString() - : scanString(); + ? scanString(TokenFlags.TripleQuoted) + : scanString(TokenFlags.None); case CharCode.Exclamation: return lookAhead(1) === CharCode.Equals @@ -656,10 +663,10 @@ export function createScanner( return (token = Token.EndOfFile); } - function scanStringTemplate(): StringTemplateToken { - tokenPosition = position; + function reScanStringTemplate(lastTokenFlags: TokenFlags): StringTemplateToken { + position = tokenPosition; tokenFlags = TokenFlags.None; - return scanStringTemplateSpan(); + return scanStringTemplateSpan(lastTokenFlags); } function scanRange(range: TextRange, callback: () => T): T { @@ -840,91 +847,112 @@ export function createScanner( return unterminated(Token.DocCodeSpan); } - function scanString(): Token.StringLiteral | Token.StringTemplateHead { - position++; // consume '"' - - loop: for (; !eof(); position++) { - const ch = input.charCodeAt(position); - switch (ch) { - case CharCode.Backslash: - tokenFlags |= TokenFlags.Escaped; - position++; - if (eof()) { - break loop; - } - continue; - case CharCode.DoubleQuote: - position++; - return (token = Token.StringLiteral); - case CharCode.$: - if (lookAhead(1) === CharCode.OpenBrace) { - position += 2; - return (token = Token.StringTemplateHead); - } - continue; - case CharCode.CarriageReturn: - case CharCode.LineFeed: - break loop; - } + function scanString(tokenFlags: TokenFlags): Token.StringLiteral | Token.StringTemplateHead { + if (tokenFlags & TokenFlags.TripleQuoted) { + position += 3; // consume '"""' + } else { + position++; // consume '"' } - return unterminated(Token.StringLiteral); + return scanStringLiteralLike(tokenFlags, Token.StringTemplateHead, Token.StringLiteral); } - function scanStringTemplateSpan(): Token.StringTemplateMiddle | Token.StringTemplateTail { + function scanStringTemplateSpan( + tokenFlags: TokenFlags + ): Token.StringTemplateMiddle | Token.StringTemplateTail { + position++; // consume '{' + + return scanStringLiteralLike(tokenFlags, Token.StringTemplateMiddle, Token.StringTemplateTail); + } + + function scanStringLiteralLike( + requestedTokenFlags: TokenFlags, + template: M, + tail: T + ): M | T { + const multiLine = requestedTokenFlags & TokenFlags.TripleQuoted; + tokenFlags = requestedTokenFlags; loop: for (; !eof(); position++) { const ch = input.charCodeAt(position); switch (ch) { case CharCode.Backslash: - tokenFlags |= TokenFlags.Escaped; + requestedTokenFlags |= TokenFlags.Escaped; position++; if (eof()) { break loop; } continue; case CharCode.DoubleQuote: - position++; - return (token = Token.StringTemplateTail); + if (multiLine) { + if (lookAhead(1) === CharCode.DoubleQuote && lookAhead(2) === CharCode.DoubleQuote) { + position += 3; + token = tail; + return tail; + } else { + continue; + } + } else { + position++; + token = tail; + return tail; + } case CharCode.$: if (lookAhead(1) === CharCode.OpenBrace) { position += 2; - return (token = Token.StringTemplateMiddle); + token = template; + return template; } continue; case CharCode.CarriageReturn: case CharCode.LineFeed: - break loop; + if (multiLine) { + continue; + } else { + break loop; + } } } - return unterminated(Token.StringTemplateTail); + return unterminated(tail); } - function scanTripleQuotedString(): Token.StringLiteral | Token.StringTemplateHead { - tokenFlags |= TokenFlags.TripleQuoted; - position += 3; // consume '"""' - - for (; !eof(); position++) { - if ( - input.charCodeAt(position) === CharCode.DoubleQuote && - lookAhead(1) === CharCode.DoubleQuote && - lookAhead(2) === CharCode.DoubleQuote - ) { - position += 3; - return (token = Token.StringLiteral); - } + function getStringLiteralOffsetStart( + token: Token.StringLiteral | StringTemplateToken, + tokenFlags: TokenFlags + ) { + switch (token) { + case Token.StringLiteral: + case Token.StringTemplateHead: + return tokenFlags & TokenFlags.TripleQuoted ? 3 : 1; // """ or " + default: + return 1; // { } + } - return unterminated(Token.StringLiteral); + function getStringLiteralOffsetEnd( + token: Token.StringLiteral | StringTemplateToken, + tokenFlags: TokenFlags + ) { + switch (token) { + case Token.StringLiteral: + case Token.StringTemplateTail: + return tokenFlags & TokenFlags.TripleQuoted ? 3 : 1; // """ or " + default: + return 2; // ${ + } } - function getStringTokenValue(): string { - const quoteLength = tokenFlags & TokenFlags.TripleQuoted ? 3 : 1; - const start = tokenPosition + quoteLength; - const end = tokenFlags & TokenFlags.Unterminated ? position : position - quoteLength; + function getStringTokenValue( + token: Token.StringLiteral | StringTemplateToken, + tokenFlags: TokenFlags + ): string { + const startOffset = getStringLiteralOffsetStart(token, tokenFlags); + const endOffset = getStringLiteralOffsetEnd(token, tokenFlags); + const start = tokenPosition + startOffset; + const end = tokenFlags & TokenFlags.Unterminated ? position : position - endOffset; if (tokenFlags & TokenFlags.TripleQuoted) { - return unindentAndUnescapeTripleQuotedString(start, end); + return unindentAndUnescapeTripleQuotedString(start, end, token); } if (tokenFlags & TokenFlags.Escaped) { @@ -951,20 +979,26 @@ export function createScanner( return text; } - function unindentAndUnescapeTripleQuotedString(start: number, end: number): string { - // ignore leading whitespace before required initial line break - while (start < end && isWhiteSpaceSingleLine(input.charCodeAt(start))) { - start++; - } + function unindentAndUnescapeTripleQuotedString( + start: number, + end: number, + token: Token.StringLiteral | StringTemplateToken + ): string { + if (token === Token.StringLiteral || token === Token.StringTemplateHead) { + // ignore leading whitespace before required initial line break + while (start < end && isWhiteSpaceSingleLine(input.charCodeAt(start))) { + start++; + } - // remove required initial line break - if (isLineBreak(input.charCodeAt(start))) { - if (isCrlf(start, start, end)) { + // remove required initial line break + if (isLineBreak(input.charCodeAt(start))) { + if (isCrlf(start, start, end)) { + start++; + } start++; + } else { + error({ code: "no-new-line-start-triple-quote" }); } - start++; - } else { - error({ code: "no-new-line-start-triple-quote" }); } // remove whitespace before closing delimiter and record it as required @@ -975,14 +1009,16 @@ export function createScanner( } const indentationStart = end; - // remove required final line break - if (isLineBreak(input.charCodeAt(end - 1))) { - if (isCrlf(end - 2, start, end)) { + if (token === Token.StringLiteral || token === Token.StringTemplateTail) { + // remove required final line break + if (isLineBreak(input.charCodeAt(end - 1))) { + if (isCrlf(end - 2, start, end)) { + end--; + } end--; + } else { + error({ code: "no-new-line-end-triple-quote" }); } - end--; - } else { - error({ code: "no-new-line-end-triple-quote" }); } // remove required matching indentation from each line and unescape in the diff --git a/packages/compiler/src/core/types.ts b/packages/compiler/src/core/types.ts index b4b9722593..41c7e36c2e 100644 --- a/packages/compiler/src/core/types.ts +++ b/packages/compiler/src/core/types.ts @@ -4,6 +4,7 @@ import { AssetEmitter } from "../emitter-framework/types.js"; import { YamlScript } from "../yaml/types.js"; import { ModuleResolutionResult } from "./module-resolver.js"; import { Program } from "./program.js"; +import type { TokenFlags } from "./scanner.js"; // prettier-ignore export type MarshalledValue = @@ -1265,6 +1266,9 @@ export interface StringTemplateSpanNode extends BaseNode { export interface StringTemplateLiteralLikeNode extends BaseNode { text: string; + + /** @internal */ + tokenFlags: TokenFlags; } export interface StringTemplateHeadNode extends StringTemplateLiteralLikeNode { diff --git a/packages/compiler/test/parser.test.ts b/packages/compiler/test/parser.test.ts index f35c3768bf..cffec5a7dd 100644 --- a/packages/compiler/test/parser.test.ts +++ b/packages/compiler/test/parser.test.ts @@ -8,11 +8,16 @@ import { NodeFlags, ParseOptions, SourceFile, + StringTemplateExpressionNode, SyntaxKind, TypeSpecScriptNode, } from "../src/core/types.js"; import { DecorableNode } from "../src/formatter/print/types.js"; -import { DiagnosticMatch, expectDiagnostics } from "../src/testing/expect.js"; +import { + DiagnosticMatch, + expectDiagnosticEmpty, + expectDiagnostics, +} from "../src/testing/expect.js"; describe("compiler: parser", () => { describe("import statements", () => { @@ -518,6 +523,54 @@ describe("compiler: parser", () => { parseErrorEach(bad.map((e) => [`model ${e[0]} {}`, [e[1]]])); }); + describe("string template expressions", () => { + function getStringTemplateNode(astNode: TypeSpecScriptNode): StringTemplateExpressionNode { + const statement = astNode.statements[0]; + strictEqual(statement.kind, SyntaxKind.AliasStatement); + const node = statement.value; + strictEqual(node.kind, SyntaxKind.StringTemplateExpression); + return node; + } + + it("parse a single line template expression", () => { + const astNode = parseSuccessWithLog(`alias T = "Start \${"one"} middle \${23} end";`); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.text, "Start "); + strictEqual(node.spans.length, 2); + + const span0 = node.spans[0]; + strictEqual(span0.literal.text, " middle "); + strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); + strictEqual(span0.expression.value, "one"); + + const span1 = node.spans[1]; + strictEqual(span1.literal.text, " end"); + strictEqual(span1.expression.kind, SyntaxKind.NumericLiteral); + strictEqual(span1.expression.value, 23); + }); + + it("parse a multiple line template expression", () => { + const astNode = parseSuccessWithLog(`alias T = """ + Start \${"one"} + middle \${23} + end + """;`); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.text, "Start "); + strictEqual(node.spans.length, 2); + + const span0 = node.spans[0]; + strictEqual(span0.literal.text, " middle "); + strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); + strictEqual(span0.expression.value, "one"); + + const span1 = node.spans[1]; + strictEqual(span1.literal.text, " end"); + strictEqual(span1.expression.kind, SyntaxKind.NumericLiteral); + strictEqual(span1.expression.value, 23); + }); + }); + // smaller repro of previous regen-samples baseline failures describe("sample regressions", () => { parseEach([ @@ -1224,6 +1277,33 @@ function checkPositioning(node: Node, file: SourceFile) { }); } +/** + * Parse the given code and log debug information. + */ +function parseWithLog(code: string, options?: ParseOptions): TypeSpecScriptNode { + logVerboseTestOutput("=== Source ==="); + logVerboseTestOutput(code); + + const astNode = parse(code, options); + logVerboseTestOutput("\n=== Parse Result ==="); + dumpAST(astNode); + return astNode; +} +/** + * Check the given code parse successfully and log debug information. + */ +function parseSuccessWithLog(code: string, options?: ParseOptions): TypeSpecScriptNode { + const astNode = parseWithLog(code, options); + logVerboseTestOutput("\n=== Diagnostics ==="); + logVerboseTestOutput((log) => { + for (const each of astNode.parseDiagnostics) { + log(formatDiagnostic(each)); + } + }); + expectDiagnosticEmpty(astNode.parseDiagnostics); + return astNode; +} + /** * * @param cases Test cases @@ -1237,16 +1317,10 @@ function parseErrorEach( ) { for (const [code, matches, callback] of cases) { it(`doesn't parse '${shorten(code)}'`, () => { - logVerboseTestOutput("=== Source ==="); - logVerboseTestOutput(code); - - const astNode = parse(code, options); + const astNode = parseWithLog(code, options); if (callback) { callback(astNode); } - logVerboseTestOutput("\n=== Parse Result ==="); - dumpAST(astNode); - logVerboseTestOutput("\n=== Diagnostics ==="); logVerboseTestOutput((log) => { for (const each of astNode.parseDiagnostics) { From e13b6f5ebde4dcf43ae09bfbdff7c55e3979381d Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 2 Nov 2023 15:58:04 -0700 Subject: [PATCH 03/24] Unindent after --- packages/compiler/src/core/parser.ts | 17 +++-- packages/compiler/src/core/scanner.ts | 91 +++++++++++++++++++++------ packages/compiler/src/core/types.ts | 5 +- 3 files changed, 87 insertions(+), 26 deletions(-) diff --git a/packages/compiler/src/core/parser.ts b/packages/compiler/src/core/parser.ts index f1b28850ad..d75c8980e2 100644 --- a/packages/compiler/src/core/parser.ts +++ b/packages/compiler/src/core/parser.ts @@ -1457,6 +1457,12 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa const pos = tokenPos(); const head = parseStringTemplateHead(); const spans = parseStringTemplateSpans(head.tokenFlags); + const last = spans[spans.length - 1]; + const indent = scanner.findTripleQuotedStringIndent(last.literal.rawText); + mutate(head).text = scanner.unindentTripleQuotedString(head.rawText, indent); + for (const span of spans) { + mutate(span.literal).text = scanner.unindentTripleQuotedString(span.literal.rawText, indent); + } return { kind: SyntaxKind.StringTemplateExpression, head, @@ -1467,13 +1473,14 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa function parseStringTemplateHead(): StringTemplateHeadNode { const pos = tokenPos(); - const text = tokenValue(); + const rawText = tokenValue(); const flags = tokenFlags(); parseExpected(Token.StringTemplateHead); return { kind: SyntaxKind.StringTemplateHead, - text, + rawText, + text: "", tokenFlags: flags, ...finishNode(pos), }; @@ -1511,6 +1518,7 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa parseExpected(Token.StringTemplateTail); return { kind: SyntaxKind.StringTemplateTail, + rawText: "", text: "", tokenFlags: tokenFlags(), ...finishNode(pos), @@ -1520,7 +1528,7 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa function parseTemplateMiddleOrTemplateTail(): StringTemplateMiddleNode | StringTemplateTailNode { const pos = tokenPos(); - const text = tokenValue(); + const rawText = tokenValue(); const kind = token() === Token.StringTemplateMiddle ? SyntaxKind.StringTemplateMiddle @@ -1529,7 +1537,8 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa nextToken(); return { kind, - text, + rawText, + text: "", tokenFlags: tokenFlags(), ...finishNode(pos), }; diff --git a/packages/compiler/src/core/scanner.ts b/packages/compiler/src/core/scanner.ts index 2898e6150b..daeb5c8391 100644 --- a/packages/compiler/src/core/scanner.ts +++ b/packages/compiler/src/core/scanner.ts @@ -315,6 +315,17 @@ export interface Scanner { */ reScanStringTemplate(tokenFlags: TokenFlags): StringTemplateToken; + /** + * Finds the indent for the given triple quoted string. + * @param input Triple quoted string rawText. + */ + findTripleQuotedStringIndent(input: string): string; + + /** + * Unindent the triple quoted string rawText + */ + unindentTripleQuotedString(input: string, indent: string): string; + /** Reset the scanner to the given start and end positions, invoke the callback, and then restore scanner state. */ scanRange(range: TextRange, callback: () => T): T; @@ -402,6 +413,8 @@ export function createScanner( scanRange, scanDoc, reScanStringTemplate, + findTripleQuotedStringIndent, + unindentTripleQuotedString, eof, getTokenText, getTokenValue, @@ -952,7 +965,7 @@ export function createScanner( const end = tokenFlags & TokenFlags.Unterminated ? position : position - endOffset; if (tokenFlags & TokenFlags.TripleQuoted) { - return unindentAndUnescapeTripleQuotedString(start, end, token); + return unescapeTripleQuotedString(start, end, token); } if (tokenFlags & TokenFlags.Escaped) { @@ -979,7 +992,7 @@ export function createScanner( return text; } - function unindentAndUnescapeTripleQuotedString( + function unescapeTripleQuotedString( start: number, end: number, token: Token.StringLiteral | StringTemplateToken @@ -1001,14 +1014,6 @@ export function createScanner( } } - // remove whitespace before closing delimiter and record it as required - // indentation for all lines - const indentationEnd = end; - while (end > start && isWhiteSpaceSingleLine(input.charCodeAt(end - 1))) { - end--; - } - const indentationStart = end; - if (token === Token.StringLiteral || token === Token.StringTemplateTail) { // remove required final line break if (isLineBreak(input.charCodeAt(end - 1))) { @@ -1027,7 +1032,7 @@ export function createScanner( let pos = start; while (pos < end) { // skip indentation at start of line - start = skipMatchingIndentation(pos, end, indentationStart, indentationEnd); + start = pos; let ch; while (pos < end && !isLineBreak((ch = input.charCodeAt(pos)))) { @@ -1065,6 +1070,57 @@ export function createScanner( return result; } + function findTripleQuotedStringIndent(input: string): string { + let end = input.length - 1; + // remove whitespace before closing delimiter and record it as required + // indentation for all lines + const indentationEnd = end; + while (end > 0 && isWhiteSpaceSingleLine(input.charCodeAt(end - 1))) { + end--; + } + const indentationStart = end; + + // remove required final line break + if (isLineBreak(input.charCodeAt(end - 1))) { + if (isCrlf(end - 2, 0, end)) { + end--; + } + end--; + } else { + error({ code: "no-new-line-end-triple-quote" }); + } + + return input.substring(indentationStart, indentationEnd); + } + + function unindentTripleQuotedString(input: string, indent: string): string { + let start = 0; + const end = input.length - 1; + + // remove required matching indentation from each line and unescape in the + // process of doing so + let result = ""; + let pos = start; + while (pos < end) { + // skip indentation at start of line + start = skipMatchingIndentation(pos, end, indent); + + while (pos < end && !isLineBreak(input.charCodeAt(pos))) { + pos++; + continue; + } + + if (pos < end) { + pos++; + result += input.substring(start, pos); + start = pos; + } + } + + result += input.substring(start, pos); + return result; + } + function isCrlf(pos: number, start: number, end: number) { return ( pos >= start && @@ -1074,14 +1130,9 @@ export function createScanner( ); } - function skipMatchingIndentation( - pos: number, - end: number, - indentationStart: number, - indentationEnd: number - ): number { - let indentationPos = indentationStart; - end = Math.min(end, pos + (indentationEnd - indentationStart)); + function skipMatchingIndentation(pos: number, end: number, indent: string): number { + let indentationPos = 0; + end = Math.min(end, pos + indent.length); while (pos < end) { const ch = input.charCodeAt(pos); @@ -1089,7 +1140,7 @@ export function createScanner( // allow subset of indentation if line has only whitespace break; } - if (ch !== input.charCodeAt(indentationPos)) { + if (ch !== indent.charCodeAt(indentationPos)) { error({ code: "triple-quote-indent" }); break; } diff --git a/packages/compiler/src/core/types.ts b/packages/compiler/src/core/types.ts index 41c7e36c2e..039ed84d81 100644 --- a/packages/compiler/src/core/types.ts +++ b/packages/compiler/src/core/types.ts @@ -1265,10 +1265,11 @@ export interface StringTemplateSpanNode extends BaseNode { } export interface StringTemplateLiteralLikeNode extends BaseNode { - text: string; + readonly rawText: string; + readonly text: string; /** @internal */ - tokenFlags: TokenFlags; + readonly tokenFlags: TokenFlags; } export interface StringTemplateHeadNode extends StringTemplateLiteralLikeNode { From ab49c95d3cccaea29d8f45c68bba194b9652ad60 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 3 Nov 2023 10:26:35 -0700 Subject: [PATCH 04/24] Scanner and parser working --- packages/compiler/src/core/parser.ts | 50 +++++--- packages/compiler/src/core/scanner.ts | 173 ++++++++++++++------------ packages/compiler/src/core/types.ts | 2 +- packages/compiler/test/parser.test.ts | 117 ++++++++++++----- 4 files changed, 218 insertions(+), 124 deletions(-) diff --git a/packages/compiler/src/core/parser.ts b/packages/compiler/src/core/parser.ts index d75c8980e2..f600752d07 100644 --- a/packages/compiler/src/core/parser.ts +++ b/packages/compiler/src/core/parser.ts @@ -1458,10 +1458,30 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa const head = parseStringTemplateHead(); const spans = parseStringTemplateSpans(head.tokenFlags); const last = spans[spans.length - 1]; - const indent = scanner.findTripleQuotedStringIndent(last.literal.rawText); - mutate(head).text = scanner.unindentTripleQuotedString(head.rawText, indent); - for (const span of spans) { - mutate(span.literal).text = scanner.unindentTripleQuotedString(span.literal.rawText, indent); + + if (head.tokenFlags & TokenFlags.TripleQuoted) { + const [indentationsStart, indentationEnd] = scanner.findTripleQuotedStringIndent( + last.literal.pos, + last.literal.end + ); + mutate(head).text = scanner.unindentAndUnescapeTripleQuotedString( + head.pos, + head.end, + indentationsStart, + indentationEnd, + Token.StringTemplateHead, + head.tokenFlags + ); + for (const span of spans) { + mutate(span.literal).text = scanner.unindentAndUnescapeTripleQuotedString( + span.literal.pos, + span.literal.end, + indentationsStart, + indentationEnd, + span === last ? Token.StringTemplateTail : Token.StringTemplateMiddle, + head.tokenFlags + ); + } } return { kind: SyntaxKind.StringTemplateExpression, @@ -1473,14 +1493,14 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa function parseStringTemplateHead(): StringTemplateHeadNode { const pos = tokenPos(); - const rawText = tokenValue(); const flags = tokenFlags(); + const text = flags & TokenFlags.TripleQuoted ? "" : tokenValue(); + parseExpected(Token.StringTemplateHead); return { kind: SyntaxKind.StringTemplateHead, - rawText, - text: "", + text, tokenFlags: flags, ...finishNode(pos), }; @@ -1511,6 +1531,9 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa headTokenFlags: TokenFlags ): StringTemplateMiddleNode | StringTemplateTailNode { const pos = tokenPos(); + const flags = tokenFlags(); + const text = flags & TokenFlags.TripleQuoted ? "" : tokenValue(); + if (token() === Token.CloseBrace) { nextStringTemplateToken(headTokenFlags); return parseTemplateMiddleOrTemplateTail(); @@ -1518,9 +1541,8 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa parseExpected(Token.StringTemplateTail); return { kind: SyntaxKind.StringTemplateTail, - rawText: "", - text: "", - tokenFlags: tokenFlags(), + text, + tokenFlags: flags, ...finishNode(pos), }; } @@ -1528,7 +1550,8 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa function parseTemplateMiddleOrTemplateTail(): StringTemplateMiddleNode | StringTemplateTailNode { const pos = tokenPos(); - const rawText = tokenValue(); + const flags = tokenFlags(); + const text = flags & TokenFlags.TripleQuoted ? "" : tokenValue(); const kind = token() === Token.StringTemplateMiddle ? SyntaxKind.StringTemplateMiddle @@ -1537,9 +1560,8 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa nextToken(); return { kind, - rawText, - text: "", - tokenFlags: tokenFlags(), + text, + tokenFlags: flags, ...finishNode(pos), }; } diff --git a/packages/compiler/src/core/scanner.ts b/packages/compiler/src/core/scanner.ts index daeb5c8391..46354e1da1 100644 --- a/packages/compiler/src/core/scanner.ts +++ b/packages/compiler/src/core/scanner.ts @@ -317,14 +317,22 @@ export interface Scanner { /** * Finds the indent for the given triple quoted string. - * @param input Triple quoted string rawText. + * @param start + * @param end */ - findTripleQuotedStringIndent(input: string): string; + findTripleQuotedStringIndent(start: number, end: number): [number, number]; /** - * Unindent the triple quoted string rawText + * Unindent and unescape the triple quoted string rawText */ - unindentTripleQuotedString(input: string, indent: string): string; + unindentAndUnescapeTripleQuotedString( + start: number, + end: number, + indentationStart: number, + indentationEnd: number, + token: Token.StringLiteral | StringTemplateToken, + tokenFlags: TokenFlags + ): string; /** Reset the scanner to the given start and end positions, invoke the callback, and then restore scanner state. */ scanRange(range: TextRange, callback: () => T): T; @@ -414,7 +422,7 @@ export function createScanner( scanDoc, reScanStringTemplate, findTripleQuotedStringIndent, - unindentTripleQuotedString, + unindentAndUnescapeTripleQuotedString, eof, getTokenText, getTokenValue, @@ -747,10 +755,14 @@ export function createScanner( function error< C extends keyof CompilerDiagnostics, M extends keyof CompilerDiagnostics[C] = "default", - >(report: Omit, "target">) { + >( + report: Omit, "target">, + pos?: number, + end?: number + ) { const diagnostic = createDiagnostic({ ...report, - target: { file, pos: tokenPosition, end: position }, + target: { file, pos: pos ?? tokenPosition, end: end ?? position }, } as any); diagnosticHandler(diagnostic); } @@ -889,7 +901,7 @@ export function createScanner( const ch = input.charCodeAt(position); switch (ch) { case CharCode.Backslash: - requestedTokenFlags |= TokenFlags.Escaped; + tokenFlags |= TokenFlags.Escaped; position++; if (eof()) { break loop; @@ -959,15 +971,25 @@ export function createScanner( token: Token.StringLiteral | StringTemplateToken, tokenFlags: TokenFlags ): string { + if (tokenFlags & TokenFlags.TripleQuoted) { + const start = tokenPosition; + const end = position; + const [indentationStart, indentationEnd] = findTripleQuotedStringIndent(start, end); + return unindentAndUnescapeTripleQuotedString( + start, + end, + indentationStart, + indentationEnd, + token, + tokenFlags + ); + } + const startOffset = getStringLiteralOffsetStart(token, tokenFlags); const endOffset = getStringLiteralOffsetEnd(token, tokenFlags); const start = tokenPosition + startOffset; const end = tokenFlags & TokenFlags.Unterminated ? position : position - endOffset; - if (tokenFlags & TokenFlags.TripleQuoted) { - return unescapeTripleQuotedString(start, end, token); - } - if (tokenFlags & TokenFlags.Escaped) { return unescapeString(start, end); } @@ -992,17 +1014,47 @@ export function createScanner( return text; } - function unescapeTripleQuotedString( + function findTripleQuotedStringIndent(start: number, end: number): [number, number] { + end = end - 3; // Remove the """ + // remove whitespace before closing delimiter and record it as required + // indentation for all lines + const indentationEnd = end; + while (end > start && isWhiteSpaceSingleLine(input.charCodeAt(end - 1))) { + end--; + } + const indentationStart = end; + + // remove required final line break + if (isLineBreak(input.charCodeAt(end - 1))) { + if (isCrlf(end - 2, 0, end)) { + end--; + } + end--; + } else { + error({ code: "no-new-line-end-triple-quote" }); + } + + return [indentationStart, indentationEnd]; + } + + function unindentAndUnescapeTripleQuotedString( start: number, end: number, - token: Token.StringLiteral | StringTemplateToken + indentationStart: number, + indentationEnd: number, + token: Token.StringLiteral | StringTemplateToken, + tokenFlags: TokenFlags ): string { + const startOffset = getStringLiteralOffsetStart(token, tokenFlags); + const endOffset = getStringLiteralOffsetEnd(token, tokenFlags); + start = start + startOffset; + end = tokenFlags & TokenFlags.Unterminated ? end : end - endOffset; + if (token === Token.StringLiteral || token === Token.StringTemplateHead) { // ignore leading whitespace before required initial line break while (start < end && isWhiteSpaceSingleLine(input.charCodeAt(start))) { start++; } - // remove required initial line break if (isLineBreak(input.charCodeAt(start))) { if (isCrlf(start, start, end)) { @@ -1015,6 +1067,10 @@ export function createScanner( } if (token === Token.StringLiteral || token === Token.StringTemplateTail) { + while (end > start && isWhiteSpaceSingleLine(input.charCodeAt(end - 1))) { + end--; + } + // remove required final line break if (isLineBreak(input.charCodeAt(end - 1))) { if (isCrlf(end - 2, start, end)) { @@ -1026,13 +1082,22 @@ export function createScanner( } } + let skipUnindentOnce = false; + // We are resuming from the middle of a line so we want to keep text as it is from there. + if (token === Token.StringTemplateMiddle || token === Token.StringTemplateTail) { + skipUnindentOnce = true; + } // remove required matching indentation from each line and unescape in the // process of doing so let result = ""; let pos = start; while (pos < end) { - // skip indentation at start of line - start = pos; + if (skipUnindentOnce) { + skipUnindentOnce = false; + } else { + // skip indentation at start of line + start = skipMatchingIndentation(pos, end, indentationStart, indentationEnd); + } let ch; while (pos < end && !isLineBreak((ch = input.charCodeAt(pos)))) { @@ -1042,7 +1107,7 @@ export function createScanner( } result += input.substring(start, pos); if (pos === end - 1) { - error({ code: "invalid-escape-sequence" }); + error({ code: "invalid-escape-sequence" }, pos, pos); pos++; } else { result += unescapeOne(pos); @@ -1050,7 +1115,6 @@ export function createScanner( } start = pos; } - if (pos < end) { if (isCrlf(pos, start, end)) { // CRLF in multi-line string is normalized to LF in string value. @@ -1065,58 +1129,6 @@ export function createScanner( start = pos; } } - - result += input.substring(start, pos); - return result; - } - - function findTripleQuotedStringIndent(input: string): string { - let end = input.length - 1; - // remove whitespace before closing delimiter and record it as required - // indentation for all lines - const indentationEnd = end; - while (end > 0 && isWhiteSpaceSingleLine(input.charCodeAt(end - 1))) { - end--; - } - const indentationStart = end; - - // remove required final line break - if (isLineBreak(input.charCodeAt(end - 1))) { - if (isCrlf(end - 2, 0, end)) { - end--; - } - end--; - } else { - error({ code: "no-new-line-end-triple-quote" }); - } - - return input.substring(indentationStart, indentationEnd); - } - - function unindentTripleQuotedString(input: string, indent: string): string { - let start = 0; - const end = input.length - 1; - - // remove required matching indentation from each line and unescape in the - // process of doing so - let result = ""; - let pos = start; - while (pos < end) { - // skip indentation at start of line - start = skipMatchingIndentation(pos, end, indent); - - while (pos < end && !isLineBreak(input.charCodeAt(pos))) { - pos++; - continue; - } - - if (pos < end) { - pos++; - result += input.substring(start, pos); - start = pos; - } - } - result += input.substring(start, pos); return result; } @@ -1130,9 +1142,14 @@ export function createScanner( ); } - function skipMatchingIndentation(pos: number, end: number, indent: string): number { - let indentationPos = 0; - end = Math.min(end, pos + indent.length); + function skipMatchingIndentation( + pos: number, + end: number, + indentationStart: number, + indentationEnd: number + ): number { + let indentationPos = indentationStart; + end = Math.min(end, pos + (indentationEnd - indentationStart)); while (pos < end) { const ch = input.charCodeAt(pos); @@ -1140,7 +1157,7 @@ export function createScanner( // allow subset of indentation if line has only whitespace break; } - if (ch !== indent.charCodeAt(indentationPos)) { + if (ch !== input.charCodeAt(indentationPos)) { error({ code: "triple-quote-indent" }); break; } @@ -1163,7 +1180,7 @@ export function createScanner( } if (pos === end - 1) { - error({ code: "invalid-escape-sequence" }); + error({ code: "invalid-escape-sequence" }, pos, pos); break; } @@ -1190,10 +1207,12 @@ export function createScanner( return '"'; case CharCode.Backslash: return "\\"; + case CharCode.$: + return "$"; case CharCode.Backtick: return "`"; default: - error({ code: "invalid-escape-sequence" }); + error({ code: "invalid-escape-sequence" }, pos, pos + 2); return String.fromCharCode(ch); } } diff --git a/packages/compiler/src/core/types.ts b/packages/compiler/src/core/types.ts index 039ed84d81..18d89aca07 100644 --- a/packages/compiler/src/core/types.ts +++ b/packages/compiler/src/core/types.ts @@ -1265,7 +1265,7 @@ export interface StringTemplateSpanNode extends BaseNode { } export interface StringTemplateLiteralLikeNode extends BaseNode { - readonly rawText: string; + // TODO-TIM? should it be value to be inline with other literals? readonly text: string; /** @internal */ diff --git a/packages/compiler/test/parser.test.ts b/packages/compiler/test/parser.test.ts index cffec5a7dd..bce00e484a 100644 --- a/packages/compiler/test/parser.test.ts +++ b/packages/compiler/test/parser.test.ts @@ -524,50 +524,103 @@ describe("compiler: parser", () => { }); describe("string template expressions", () => { - function getStringTemplateNode(astNode: TypeSpecScriptNode): StringTemplateExpressionNode { + function getNode(astNode: TypeSpecScriptNode): Node { const statement = astNode.statements[0]; strictEqual(statement.kind, SyntaxKind.AliasStatement); - const node = statement.value; + return statement.value; + } + function getStringTemplateNode(astNode: TypeSpecScriptNode): StringTemplateExpressionNode { + const node = getNode(astNode); strictEqual(node.kind, SyntaxKind.StringTemplateExpression); return node; } - it("parse a single line template expression", () => { - const astNode = parseSuccessWithLog(`alias T = "Start \${"one"} middle \${23} end";`); - const node = getStringTemplateNode(astNode); - strictEqual(node.head.text, "Start "); - strictEqual(node.spans.length, 2); - - const span0 = node.spans[0]; - strictEqual(span0.literal.text, " middle "); - strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); - strictEqual(span0.expression.value, "one"); - - const span1 = node.spans[1]; - strictEqual(span1.literal.text, " end"); - strictEqual(span1.expression.kind, SyntaxKind.NumericLiteral); - strictEqual(span1.expression.value, 23); + describe("single line", () => { + it("parse a single line template expression", () => { + const astNode = parseSuccessWithLog(`alias T = "Start \${"one"} middle \${23} end";`); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.text, "Start "); + strictEqual(node.spans.length, 2); + + const span0 = node.spans[0]; + strictEqual(span0.literal.text, " middle "); + strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); + strictEqual(span0.expression.value, "one"); + + const span1 = node.spans[1]; + strictEqual(span1.literal.text, " end"); + strictEqual(span1.expression.kind, SyntaxKind.NumericLiteral); + strictEqual(span1.expression.value, 23); + }); + + it("can escape some ${}", () => { + const astNode = parseSuccessWithLog(`alias T = "Start \${"one"} middle \\\${23} end";`); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.text, "Start "); + strictEqual(node.spans.length, 1); + + const span0 = node.spans[0]; + strictEqual(span0.literal.text, " middle ${23} end"); + strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); + strictEqual(span0.expression.value, "one"); + }); + + it("string with all ${} escape is still a StringLiteral", () => { + const astNode = parseSuccessWithLog(`alias T = "Start \\\${12} middle \\\${23} end";`); + const node = getNode(astNode); + strictEqual(node.kind, SyntaxKind.StringLiteral); + strictEqual(node.value, "Start ${12} middle ${23} end"); + }); }); - it("parse a multiple line template expression", () => { - const astNode = parseSuccessWithLog(`alias T = """ + describe("multi line", () => { + it("parse a multiple line template expression", () => { + const astNode = parseSuccessWithLog(`alias T = """ Start \${"one"} middle \${23} end """;`); - const node = getStringTemplateNode(astNode); - strictEqual(node.head.text, "Start "); - strictEqual(node.spans.length, 2); - - const span0 = node.spans[0]; - strictEqual(span0.literal.text, " middle "); - strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); - strictEqual(span0.expression.value, "one"); - - const span1 = node.spans[1]; - strictEqual(span1.literal.text, " end"); - strictEqual(span1.expression.kind, SyntaxKind.NumericLiteral); - strictEqual(span1.expression.value, 23); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.text, "Start "); + strictEqual(node.spans.length, 2); + + const span0 = node.spans[0]; + strictEqual(span0.literal.text, " \nmiddle "); + strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); + strictEqual(span0.expression.value, "one"); + + const span1 = node.spans[1]; + strictEqual(span1.literal.text, " \nend"); + strictEqual(span1.expression.kind, SyntaxKind.NumericLiteral); + strictEqual(span1.expression.value, 23); + }); + + it("can escape some ${}", () => { + const astNode = parseSuccessWithLog(`alias T = """ + Start \${"one"} + middle \\\${23} + end + """;`); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.text, "Start "); + strictEqual(node.spans.length, 1); + + const span0 = node.spans[0]; + strictEqual(span0.literal.text, " \nmiddle ${23} \nend"); + strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); + strictEqual(span0.expression.value, "one"); + }); + + it("escaping all ${} still produce a string literal", () => { + const astNode = parseSuccessWithLog(`alias T = """ + Start \\\${12} + middle \\\${23} + end + """;`); + const node = getNode(astNode); + strictEqual(node.kind, SyntaxKind.StringLiteral); + strictEqual(node.value, "Start ${12} \nmiddle ${23} \nend"); + }); }); }); From 2eb4eb7acf131f93b281fecde6b037e3725c65f7 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 3 Nov 2023 11:01:09 -0700 Subject: [PATCH 05/24] Checker --- packages/compiler/src/core/checker.ts | 47 +++++++++++++++++-- packages/compiler/src/core/parser.ts | 10 ++-- packages/compiler/src/core/semantic-walker.ts | 12 +++++ packages/compiler/src/core/types.ts | 18 +++++-- packages/compiler/test/parser.test.ts | 20 ++++---- 5 files changed, 85 insertions(+), 22 deletions(-) diff --git a/packages/compiler/src/core/checker.ts b/packages/compiler/src/core/checker.ts index f07dbe06a3..57588c29f6 100644 --- a/packages/compiler/src/core/checker.ts +++ b/packages/compiler/src/core/checker.ts @@ -102,6 +102,11 @@ import { StdTypes, StringLiteral, StringLiteralNode, + StringTemplate, + StringTemplateExpressionNode, + StringTemplateHeadNode, + StringTemplateMiddleNode, + StringTemplateTailNode, Sym, SymbolFlags, SymbolLinks, @@ -641,6 +646,8 @@ export function createChecker(program: Program): Checker { return checkTupleExpression(node, mapper); case SyntaxKind.StringLiteral: return checkStringLiteral(node); + case SyntaxKind.StringTemplateExpression: + return checkStringTemplateExpresion(node, mapper); case SyntaxKind.ArrayExpression: return checkArrayExpression(node, mapper); case SyntaxKind.UnionExpression: @@ -2382,6 +2389,25 @@ export function createChecker(program: Program): Checker { return getMergedSymbol(aliasType.node!.symbol) ?? aliasSymbol; } } + + function checkStringTemplateExpresion( + node: StringTemplateExpressionNode, + mapper: TypeMapper | undefined + ): StringTemplate { + const spans: Type[] = [getLiteralType(node.head)]; + for (const span of node.spans) { + spans.push(getTypeForNode(span.expression, mapper)); + spans.push(getLiteralType(node.head)); + } + const type = createType({ + kind: "StringTemplate", + node, + spans, + }); + + return type; + } + function checkStringLiteral(str: StringLiteralNode): StringLiteral { return getLiteralType(str); } @@ -4058,7 +4084,13 @@ export function createChecker(program: Program): Checker { return finishTypeForProgramAndChecker(program, typePrototype, typeDef); } - function getLiteralType(node: StringLiteralNode): StringLiteral; + function getLiteralType( + node: + | StringLiteralNode + | StringTemplateHeadNode + | StringTemplateMiddleNode + | StringTemplateTailNode + ): StringLiteral; function getLiteralType(node: NumericLiteralNode): NumericLiteral; function getLiteralType(node: BooleanLiteralNode): BooleanLiteral; function getLiteralType(node: LiteralNode): LiteralType; @@ -4870,16 +4902,23 @@ export function createChecker(program: Program): Checker { } as const); } - function createLiteralType(value: string, node?: StringLiteralNode): StringLiteral; + function createLiteralType( + value: string, + node?: + | StringLiteralNode + | StringTemplateHeadNode + | StringTemplateMiddleNode + | StringTemplateTailNode + ): StringLiteral; function createLiteralType(value: number, node?: NumericLiteralNode): NumericLiteral; function createLiteralType(value: boolean, node?: BooleanLiteralNode): BooleanLiteral; function createLiteralType( value: string | number | boolean, - node?: StringLiteralNode | NumericLiteralNode | BooleanLiteralNode + node?: LiteralNode ): StringLiteral | NumericLiteral | BooleanLiteral; function createLiteralType( value: string | number | boolean, - node?: StringLiteralNode | NumericLiteralNode | BooleanLiteralNode + node?: LiteralNode ): StringLiteral | NumericLiteral | BooleanLiteral { if (program.literalTypes.has(value)) { return program.literalTypes.get(value)!; diff --git a/packages/compiler/src/core/parser.ts b/packages/compiler/src/core/parser.ts index f600752d07..44b5bc4329 100644 --- a/packages/compiler/src/core/parser.ts +++ b/packages/compiler/src/core/parser.ts @@ -1464,7 +1464,7 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa last.literal.pos, last.literal.end ); - mutate(head).text = scanner.unindentAndUnescapeTripleQuotedString( + mutate(head).value = scanner.unindentAndUnescapeTripleQuotedString( head.pos, head.end, indentationsStart, @@ -1473,7 +1473,7 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa head.tokenFlags ); for (const span of spans) { - mutate(span.literal).text = scanner.unindentAndUnescapeTripleQuotedString( + mutate(span.literal).value = scanner.unindentAndUnescapeTripleQuotedString( span.literal.pos, span.literal.end, indentationsStart, @@ -1500,7 +1500,7 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa return { kind: SyntaxKind.StringTemplateHead, - text, + value: text, tokenFlags: flags, ...finishNode(pos), }; @@ -1541,7 +1541,7 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa parseExpected(Token.StringTemplateTail); return { kind: SyntaxKind.StringTemplateTail, - text, + value: text, tokenFlags: flags, ...finishNode(pos), }; @@ -1560,7 +1560,7 @@ function createParser(code: string | SourceFile, options: ParseOptions = {}): Pa nextToken(); return { kind, - text, + value: text, tokenFlags: flags, ...finishNode(pos), }; diff --git a/packages/compiler/src/core/semantic-walker.ts b/packages/compiler/src/core/semantic-walker.ts index 23de52c00d..4e638502af 100644 --- a/packages/compiler/src/core/semantic-walker.ts +++ b/packages/compiler/src/core/semantic-walker.ts @@ -11,6 +11,7 @@ import { Operation, Scalar, SemanticNodeListener, + StringTemplate, TemplateParameter, Tuple, Type, @@ -320,6 +321,15 @@ function navigateTupleType(type: Tuple, context: NavigationContext) { navigateTypeInternal(value, context); } } +function navigateStringTemplate(type: StringTemplate, context: NavigationContext) { + if (checkVisited(context.visited, type)) { + return; + } + if (context.emit("stringTemplate", type) === ListenerFlow.NoRecursion) return; + for (const value of type.spans) { + navigateTypeInternal(value, context); + } +} function navigateTemplateParameter(type: TemplateParameter, context: NavigationContext) { if (checkVisited(context.visited, type)) { @@ -357,6 +367,8 @@ function navigateTypeInternal(type: Type, context: NavigationContext) { return navigateUnionTypeVariant(type, context); case "Tuple": return navigateTupleType(type, context); + case "StringTemplate": + return navigateStringTemplate(type, context); case "TemplateParameter": return navigateTemplateParameter(type, context); case "Decorator": diff --git a/packages/compiler/src/core/types.ts b/packages/compiler/src/core/types.ts index 18d89aca07..c45c9193f4 100644 --- a/packages/compiler/src/core/types.ts +++ b/packages/compiler/src/core/types.ts @@ -99,6 +99,7 @@ export type Type = | StringLiteral | NumericLiteral | BooleanLiteral + | StringTemplate | Tuple | Union | UnionVariant @@ -479,6 +480,12 @@ export interface BooleanLiteral extends BaseType { value: boolean; } +export interface StringTemplate extends BaseType { + kind: "StringTemplate"; + node: StringTemplateExpressionNode; + spans: Type[]; +} + export interface Tuple extends BaseType { kind: "Tuple"; node: TupleExpressionNode; @@ -1232,7 +1239,13 @@ export interface ModelSpreadPropertyNode extends BaseNode { readonly parent?: ModelStatementNode | ModelExpressionNode; } -export type LiteralNode = StringLiteralNode | NumericLiteralNode | BooleanLiteralNode; +export type LiteralNode = + | StringLiteralNode + | NumericLiteralNode + | BooleanLiteralNode + | StringTemplateHeadNode + | StringTemplateMiddleNode + | StringTemplateTailNode; export interface StringLiteralNode extends BaseNode { readonly kind: SyntaxKind.StringLiteral; @@ -1265,8 +1278,7 @@ export interface StringTemplateSpanNode extends BaseNode { } export interface StringTemplateLiteralLikeNode extends BaseNode { - // TODO-TIM? should it be value to be inline with other literals? - readonly text: string; + readonly value: string; /** @internal */ readonly tokenFlags: TokenFlags; diff --git a/packages/compiler/test/parser.test.ts b/packages/compiler/test/parser.test.ts index bce00e484a..8392663bbb 100644 --- a/packages/compiler/test/parser.test.ts +++ b/packages/compiler/test/parser.test.ts @@ -539,16 +539,16 @@ describe("compiler: parser", () => { it("parse a single line template expression", () => { const astNode = parseSuccessWithLog(`alias T = "Start \${"one"} middle \${23} end";`); const node = getStringTemplateNode(astNode); - strictEqual(node.head.text, "Start "); + strictEqual(node.head.value, "Start "); strictEqual(node.spans.length, 2); const span0 = node.spans[0]; - strictEqual(span0.literal.text, " middle "); + strictEqual(span0.literal.value, " middle "); strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); strictEqual(span0.expression.value, "one"); const span1 = node.spans[1]; - strictEqual(span1.literal.text, " end"); + strictEqual(span1.literal.value, " end"); strictEqual(span1.expression.kind, SyntaxKind.NumericLiteral); strictEqual(span1.expression.value, 23); }); @@ -556,11 +556,11 @@ describe("compiler: parser", () => { it("can escape some ${}", () => { const astNode = parseSuccessWithLog(`alias T = "Start \${"one"} middle \\\${23} end";`); const node = getStringTemplateNode(astNode); - strictEqual(node.head.text, "Start "); + strictEqual(node.head.value, "Start "); strictEqual(node.spans.length, 1); const span0 = node.spans[0]; - strictEqual(span0.literal.text, " middle ${23} end"); + strictEqual(span0.literal.value, " middle ${23} end"); strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); strictEqual(span0.expression.value, "one"); }); @@ -581,16 +581,16 @@ describe("compiler: parser", () => { end """;`); const node = getStringTemplateNode(astNode); - strictEqual(node.head.text, "Start "); + strictEqual(node.head.value, "Start "); strictEqual(node.spans.length, 2); const span0 = node.spans[0]; - strictEqual(span0.literal.text, " \nmiddle "); + strictEqual(span0.literal.value, " \nmiddle "); strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); strictEqual(span0.expression.value, "one"); const span1 = node.spans[1]; - strictEqual(span1.literal.text, " \nend"); + strictEqual(span1.literal.value, " \nend"); strictEqual(span1.expression.kind, SyntaxKind.NumericLiteral); strictEqual(span1.expression.value, 23); }); @@ -602,11 +602,11 @@ describe("compiler: parser", () => { end """;`); const node = getStringTemplateNode(astNode); - strictEqual(node.head.text, "Start "); + strictEqual(node.head.value, "Start "); strictEqual(node.spans.length, 1); const span0 = node.spans[0]; - strictEqual(span0.literal.text, " \nmiddle ${23} \nend"); + strictEqual(span0.literal.value, " \nmiddle ${23} \nend"); strictEqual(span0.expression.kind, SyntaxKind.StringLiteral); strictEqual(span0.expression.value, "one"); }); From d1c5c047469fe441c6c6825e0487a51630dac411 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 3 Nov 2023 11:38:42 -0700 Subject: [PATCH 06/24] String template checker --- packages/compiler/lib/reflection.tsp | 1 + packages/compiler/src/core/checker.ts | 32 +++++++- packages/compiler/src/core/semantic-walker.ts | 10 +++ packages/compiler/src/core/types.ts | 17 ++++- .../compiler/src/server/type-signature.ts | 17 +++++ .../test/checker/string-template.test.ts | 73 +++++++++++++++++++ .../tspd/src/ref-doc/utils/type-signature.ts | 18 ++++- 7 files changed, 163 insertions(+), 5 deletions(-) create mode 100644 packages/compiler/test/checker/string-template.test.ts diff --git a/packages/compiler/lib/reflection.tsp b/packages/compiler/lib/reflection.tsp index 38876f7798..f1142b2e73 100644 --- a/packages/compiler/lib/reflection.tsp +++ b/packages/compiler/lib/reflection.tsp @@ -10,3 +10,4 @@ model Operation {} model Scalar {} model Union {} model UnionVariant {} +model StringTemplate {} diff --git a/packages/compiler/src/core/checker.ts b/packages/compiler/src/core/checker.ts index 57588c29f6..fa5a2f4b9b 100644 --- a/packages/compiler/src/core/checker.ts +++ b/packages/compiler/src/core/checker.ts @@ -106,6 +106,9 @@ import { StringTemplateExpressionNode, StringTemplateHeadNode, StringTemplateMiddleNode, + StringTemplateSpan, + StringTemplateSpanLiteral, + StringTemplateSpanValue, StringTemplateTailNode, Sym, SymbolFlags, @@ -2394,10 +2397,10 @@ export function createChecker(program: Program): Checker { node: StringTemplateExpressionNode, mapper: TypeMapper | undefined ): StringTemplate { - const spans: Type[] = [getLiteralType(node.head)]; + const spans: StringTemplateSpan[] = [createTemplateSpanLiteral(node.head)]; for (const span of node.spans) { - spans.push(getTypeForNode(span.expression, mapper)); - spans.push(getLiteralType(node.head)); + spans.push(createTemplateSpanValue(span.expression, mapper)); + spans.push(createTemplateSpanLiteral(span.literal)); } const type = createType({ kind: "StringTemplate", @@ -2408,6 +2411,29 @@ export function createChecker(program: Program): Checker { return type; } + function createTemplateSpanLiteral( + node: StringTemplateHeadNode | StringTemplateMiddleNode | StringTemplateTailNode + ): StringTemplateSpanLiteral { + return createType({ + kind: "StringTemplateSpan", + node: node, + isInterpolated: false, + type: getLiteralType(node), + }); + } + + function createTemplateSpanValue( + node: Node, + mapper: TypeMapper | undefined + ): StringTemplateSpanValue { + return createType({ + kind: "StringTemplateSpan", + node: node, + isInterpolated: true, + type: getTypeForNode(node, mapper), + }); + } + function checkStringLiteral(str: StringLiteralNode): StringLiteral { return getLiteralType(str); } diff --git a/packages/compiler/src/core/semantic-walker.ts b/packages/compiler/src/core/semantic-walker.ts index 4e638502af..d6d4a5657c 100644 --- a/packages/compiler/src/core/semantic-walker.ts +++ b/packages/compiler/src/core/semantic-walker.ts @@ -12,6 +12,7 @@ import { Scalar, SemanticNodeListener, StringTemplate, + StringTemplateSpan, TemplateParameter, Tuple, Type, @@ -330,6 +331,13 @@ function navigateStringTemplate(type: StringTemplate, context: NavigationContext navigateTypeInternal(value, context); } } +function navigateStringTemplateSpan(type: StringTemplateSpan, context: NavigationContext) { + if (checkVisited(context.visited, type)) { + return; + } + if (context.emit("stringTemplateSpan", type as any) === ListenerFlow.NoRecursion) return; + navigateTypeInternal(type.type, context); +} function navigateTemplateParameter(type: TemplateParameter, context: NavigationContext) { if (checkVisited(context.visited, type)) { @@ -369,6 +377,8 @@ function navigateTypeInternal(type: Type, context: NavigationContext) { return navigateTupleType(type, context); case "StringTemplate": return navigateStringTemplate(type, context); + case "StringTemplateSpan": + return navigateStringTemplateSpan(type, context); case "TemplateParameter": return navigateTemplateParameter(type, context); case "Decorator": diff --git a/packages/compiler/src/core/types.ts b/packages/compiler/src/core/types.ts index c45c9193f4..ff2d370e72 100644 --- a/packages/compiler/src/core/types.ts +++ b/packages/compiler/src/core/types.ts @@ -100,6 +100,7 @@ export type Type = | NumericLiteral | BooleanLiteral | StringTemplate + | StringTemplateSpan | Tuple | Union | UnionVariant @@ -483,7 +484,21 @@ export interface BooleanLiteral extends BaseType { export interface StringTemplate extends BaseType { kind: "StringTemplate"; node: StringTemplateExpressionNode; - spans: Type[]; + spans: StringTemplateSpan[]; +} + +export type StringTemplateSpan = StringTemplateSpanLiteral | StringTemplateSpanValue; + +export interface StringTemplateSpanLiteral extends BaseType { + kind: "StringTemplateSpan"; + isInterpolated: false; + type: StringLiteral; +} + +export interface StringTemplateSpanValue extends BaseType { + kind: "StringTemplateSpan"; + isInterpolated: true; + type: Type; } export interface Tuple extends BaseType { diff --git a/packages/compiler/src/server/type-signature.ts b/packages/compiler/src/server/type-signature.ts index 18a9e24007..a2625b3979 100644 --- a/packages/compiler/src/server/type-signature.ts +++ b/packages/compiler/src/server/type-signature.ts @@ -10,6 +10,7 @@ import { FunctionType, ModelProperty, Operation, + StringTemplate, Sym, SyntaxKind, Type, @@ -54,6 +55,10 @@ function getTypeSignature(type: Type | ValueType): string { return `(boolean)\n${fence(type.value ? "true" : "false")}`; case "Number": return `(number)\n${fence(type.value.toString())}`; + case "StringTemplate": + return `(string template)\n${fence(getStringTemplateSignature(type))}`; + case "StringTemplateSpan": + return `(string template span)\n${fence(getTypeName(type.type))}`; case "Intrinsic": return ""; case "FunctionParameter": @@ -105,6 +110,18 @@ function getFunctionParameterSignature(parameter: FunctionParameter) { return `${rest}${printId(parameter.name)}${optional}: ${getTypeName(parameter.type)}`; } +function getStringTemplateSignature(stringTemplate: StringTemplate) { + return ( + "`" + + [ + stringTemplate.spans.map((span) => { + return span.isInterpolated ? "${" + getTypeName(span.type) + "}" : span.type.value; + }), + ].join("") + + "`" + ); +} + function getModelPropertySignature(property: ModelProperty) { const ns = getQualifier(property.model); return `${ns}${printId(property.name)}: ${getPrintableTypeName(property.type)}`; diff --git a/packages/compiler/test/checker/string-template.test.ts b/packages/compiler/test/checker/string-template.test.ts new file mode 100644 index 0000000000..fb867ac2c3 --- /dev/null +++ b/packages/compiler/test/checker/string-template.test.ts @@ -0,0 +1,73 @@ +import { strictEqual } from "assert"; +import { Model, StringTemplate } from "../../src/index.js"; +import { BasicTestRunner, createTestRunner } from "../../src/testing/index.js"; + +describe("compiler: string templates", () => { + let runner: BasicTestRunner; + + beforeEach(async () => { + runner = await createTestRunner(); + }); + + async function compileStringTemplate( + templateString: string, + other?: string + ): Promise { + const { Test } = (await runner.compile( + ` + @test model Test { + test: ${templateString}; + } + + ${other ?? ""} + ` + )) as { Test: Model }; + + const prop = Test.properties.get("test")!.type; + + strictEqual(prop.kind, "StringTemplate"); + return prop; + } + + it("simple", async () => { + const template = await compileStringTemplate(`"Start \${123} end"`); + strictEqual(template.spans.length, 3); + strictEqual(template.spans[0].isInterpolated, false); + strictEqual(template.spans[0].type.value, "Start "); + + strictEqual(template.spans[1].isInterpolated, true); + strictEqual(template.spans[1].type.kind, "Number"); + strictEqual(template.spans[1].type.value, 123); + + strictEqual(template.spans[2].isInterpolated, false); + strictEqual(template.spans[2].type.value, " end"); + }); + + it("string interpolated are marked with isInterpolated", async () => { + const template = await compileStringTemplate(`"Start \${"interpolate"} end"`); + strictEqual(template.spans.length, 3); + strictEqual(template.spans[0].isInterpolated, false); + strictEqual(template.spans[0].type.value, "Start "); + + strictEqual(template.spans[1].isInterpolated, true); + strictEqual(template.spans[1].type.kind, "String"); + strictEqual(template.spans[1].type.value, "interpolate"); + + strictEqual(template.spans[2].isInterpolated, false); + strictEqual(template.spans[2].type.value, " end"); + }); + + it("can interpolate a model", async () => { + const template = await compileStringTemplate(`"Start \${TestModel} end"`, "model TestModel {}"); + strictEqual(template.spans.length, 3); + strictEqual(template.spans[0].isInterpolated, false); + strictEqual(template.spans[0].type.value, "Start "); + + strictEqual(template.spans[1].isInterpolated, true); + strictEqual(template.spans[1].type.kind, "Model"); + strictEqual(template.spans[1].type.name, "TestModel"); + + strictEqual(template.spans[2].isInterpolated, false); + strictEqual(template.spans[2].type.value, " end"); + }); +}); diff --git a/packages/tspd/src/ref-doc/utils/type-signature.ts b/packages/tspd/src/ref-doc/utils/type-signature.ts index 7edeea1b67..15b56f8d96 100644 --- a/packages/tspd/src/ref-doc/utils/type-signature.ts +++ b/packages/tspd/src/ref-doc/utils/type-signature.ts @@ -9,6 +9,7 @@ import { Model, ModelProperty, Operation, + StringTemplate, TemplateParameterDeclarationNode, Type, UnionVariant, @@ -46,9 +47,12 @@ export function getTypeSignature(type: Type | ValueType): string { return `(number) ${type.value.toString()}`; case "Intrinsic": return `(intrinsic) ${type.name}`; - case "FunctionParameter": return getFunctionParameterSignature(type); + case "StringTemplate": + return `(string template)\n${getStringTemplateSignature(type)}`; + case "StringTemplateSpan": + return `(string template span)\n${getTypeName(type.type)}`; case "ModelProperty": return `(model property) ${`${type.name}: ${getTypeName(type.type)}`}`; case "EnumMember": @@ -119,6 +123,18 @@ function getFunctionParameterSignature(parameter: FunctionParameter) { return `${rest}${parameter.name}${optional}: ${getTypeName(parameter.type)}`; } +function getStringTemplateSignature(stringTemplate: StringTemplate) { + return ( + "`" + + [ + stringTemplate.spans.map((span) => { + return span.isInterpolated ? "${" + getTypeName(span.type) + "}" : span.type.value; + }), + ].join("") + + "`" + ); +} + function getModelPropertySignature(property: ModelProperty) { const ns = getQualifier(property.model); return `${ns}${property.name}: ${getTypeName(property.type)}`; From 0300483fa3f14a24c4078f441c8a921bc60cc083 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 3 Nov 2023 12:06:53 -0700 Subject: [PATCH 07/24] Json Schema handle template string --- packages/compiler/src/core/checker.ts | 2 +- packages/compiler/src/core/helpers/index.ts | 1 + .../src/core/helpers/string-template-utils.ts | 39 +++++++++++ packages/compiler/src/core/messages.ts | 7 ++ packages/compiler/src/core/types.ts | 2 + .../src/emitter-framework/asset-emitter.ts | 3 + .../src/emitter-framework/type-emitter.ts | 9 +++ .../json-schema/src/json-schema-emitter.ts | 13 ++++ .../json-schema/test/string-template.test.ts | 66 +++++++++++++++++++ packages/json-schema/test/utils.ts | 12 +++- 10 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 packages/compiler/src/core/helpers/string-template-utils.ts create mode 100644 packages/json-schema/test/string-template.test.ts diff --git a/packages/compiler/src/core/checker.ts b/packages/compiler/src/core/checker.ts index fa5a2f4b9b..b682e789cd 100644 --- a/packages/compiler/src/core/checker.ts +++ b/packages/compiler/src/core/checker.ts @@ -2423,7 +2423,7 @@ export function createChecker(program: Program): Checker { } function createTemplateSpanValue( - node: Node, + node: Expression, mapper: TypeMapper | undefined ): StringTemplateSpanValue { return createType({ diff --git a/packages/compiler/src/core/helpers/index.ts b/packages/compiler/src/core/helpers/index.ts index 9745264f05..f0df08a09a 100644 --- a/packages/compiler/src/core/helpers/index.ts +++ b/packages/compiler/src/core/helpers/index.ts @@ -3,5 +3,6 @@ export { getLocationContext } from "./location-context.js"; export * from "./operation-utils.js"; export * from "./path-interpolation.js"; export * from "./projected-names-utils.js"; +export { stringTemplateToString } from "./string-template-utils.js"; export * from "./type-name-utils.js"; export * from "./usage-resolver.js"; diff --git a/packages/compiler/src/core/helpers/string-template-utils.ts b/packages/compiler/src/core/helpers/string-template-utils.ts new file mode 100644 index 0000000000..366f439ac8 --- /dev/null +++ b/packages/compiler/src/core/helpers/string-template-utils.ts @@ -0,0 +1,39 @@ +import { createDiagnostic } from "../messages.js"; +import { Diagnostic, StringTemplate } from "../types.js"; +import { getTypeName } from "./type-name-utils.js"; + +/** + * Convert a string template to a string value. + * Only literal interpolated can be converted to string. + * Otherwise diagnostics will be reported. + * + * @param stringTemplate String template to convert. + */ +export function stringTemplateToString( + stringTemplate: StringTemplate +): [string, readonly Diagnostic[]] { + const diagnostics: Diagnostic[] = []; + const result = stringTemplate.spans + .map((x) => { + if (x.isInterpolated) { + switch (x.type.kind) { + case "String": + case "Number": + case "Boolean": + return String(x.type.value); + default: + diagnostics.push( + createDiagnostic({ + code: "non-literal-string-template", + target: x.node, + }) + ); + return getTypeName(x.type); + } + } else { + return x.type.value; + } + }) + .join(""); + return [result, diagnostics]; +} diff --git a/packages/compiler/src/core/messages.ts b/packages/compiler/src/core/messages.ts index 9b37299759..1a73c3b383 100644 --- a/packages/compiler/src/core/messages.ts +++ b/packages/compiler/src/core/messages.ts @@ -586,6 +586,13 @@ const diagnostics = { "Projections are experimental - your code will need to change as this feature evolves.", }, }, + "non-literal-string-template": { + severity: "error", + messages: { + default: + "Value interpolated in this string template cannot be converted to a string. Only literal types can be automatically interpolated.", + }, + }, /** * Binder diff --git a/packages/compiler/src/core/types.ts b/packages/compiler/src/core/types.ts index ff2d370e72..06fb6d4349 100644 --- a/packages/compiler/src/core/types.ts +++ b/packages/compiler/src/core/types.ts @@ -491,12 +491,14 @@ export type StringTemplateSpan = StringTemplateSpanLiteral | StringTemplateSpanV export interface StringTemplateSpanLiteral extends BaseType { kind: "StringTemplateSpan"; + node: StringTemplateHeadNode | StringTemplateMiddleNode | StringTemplateTailNode; isInterpolated: false; type: StringLiteral; } export interface StringTemplateSpanValue extends BaseType { kind: "StringTemplateSpan"; + node: Expression; isInterpolated: true; type: Type; } diff --git a/packages/compiler/src/emitter-framework/asset-emitter.ts b/packages/compiler/src/emitter-framework/asset-emitter.ts index b6a3c85456..430b1f43f5 100644 --- a/packages/compiler/src/emitter-framework/asset-emitter.ts +++ b/packages/compiler/src/emitter-framework/asset-emitter.ts @@ -728,6 +728,8 @@ export function createAssetEmitter( return "namespace"; case "ModelProperty": return "modelPropertyLiteral"; + case "StringTemplate": + return "stringTemplate"; case "Boolean": return "booleanLiteral"; case "String": @@ -855,6 +857,7 @@ function keyHasContext(key: keyof TypeEmitter) { const noReferenceContext = new Set([ ...noContext, "booleanLiteral", + "stringTemplate", "stringLiteral", "numericLiteral", "scalarDeclaration", diff --git a/packages/compiler/src/emitter-framework/type-emitter.ts b/packages/compiler/src/emitter-framework/type-emitter.ts index 184dfcea3f..579dbcf409 100644 --- a/packages/compiler/src/emitter-framework/type-emitter.ts +++ b/packages/compiler/src/emitter-framework/type-emitter.ts @@ -15,6 +15,7 @@ import { Program, Scalar, StringLiteral, + StringTemplate, Tuple, Type, Union, @@ -460,6 +461,14 @@ export class TypeEmitter> { return this.emitter.result.none(); } + stringTemplateContext(string: StringTemplate): Context { + return {}; + } + + stringTemplate(string: StringTemplate): EmitterOutput { + return this.emitter.result.none(); + } + stringLiteralContext(string: StringLiteral): Context { return {}; } diff --git a/packages/json-schema/src/json-schema-emitter.ts b/packages/json-schema/src/json-schema-emitter.ts index 7d3a29c2ff..ee2f6329de 100644 --- a/packages/json-schema/src/json-schema-emitter.ts +++ b/packages/json-schema/src/json-schema-emitter.ts @@ -25,6 +25,8 @@ import { Program, Scalar, StringLiteral, + StringTemplate, + stringTemplateToString, Tuple, Type, typespecTypeToJson, @@ -180,6 +182,17 @@ export class JsonSchemaEmitter extends TypeEmitter, JSONSche return { type: "string", const: string.value }; } + stringTemplate(string: StringTemplate): EmitterOutput { + const [value, diagnostics] = stringTemplateToString(string); + if (diagnostics.length > 0) { + this.emitter + .getProgram() + .reportDiagnostics(diagnostics.map((x) => ({ ...x, severity: "warning" }))); + return { type: "string" }; + } + return { type: "string", const: value }; + } + numericLiteral(number: NumericLiteral): EmitterOutput { return { type: "number", const: number.value }; } diff --git a/packages/json-schema/test/string-template.test.ts b/packages/json-schema/test/string-template.test.ts new file mode 100644 index 0000000000..b221e9f848 --- /dev/null +++ b/packages/json-schema/test/string-template.test.ts @@ -0,0 +1,66 @@ +import { expectDiagnostics } from "@typespec/compiler/testing"; +import { deepStrictEqual } from "assert"; +import { emitSchema, emitSchemaWithDiagnostics } from "./utils.js"; + +describe("string templates", () => { + describe("handle interpolating literals", () => { + it("string", async () => { + const schemas = await emitSchema(` + model Test { + a: "Start \${"abc"} end", + } + `); + + deepStrictEqual(schemas["Test.json"].properties.a, { + type: "string", + const: "Start abc end", + }); + }); + + it("number", async () => { + const schemas = await emitSchema(` + model Test { + a: "Start \${123} end", + } + `); + + deepStrictEqual(schemas["Test.json"].properties.a, { + type: "string", + const: "Start 123 end", + }); + }); + + it("boolean", async () => { + const schemas = await emitSchema(` + model Test { + a: "Start \${true} end", + } + `); + + deepStrictEqual(schemas["Test.json"].properties.a, { + type: "string", + const: "Start true end", + }); + }); + }); + + it("emit diagnostics if interpolation value are not literals", async () => { + const [schemas, diagnostics] = await emitSchemaWithDiagnostics(` + model Test { + a: "Start \${Bar} end", + } + model Bar {} + `); + + deepStrictEqual(schemas["Test.json"].properties.a, { + type: "string", + }); + + expectDiagnostics(diagnostics, { + code: "non-literal-string-template", + severity: "warning", + message: + "Value interpolated in this string template cannot be converted to a string. Only literal types can be automatically interpolated.", + }); + }); +}); diff --git a/packages/json-schema/test/utils.ts b/packages/json-schema/test/utils.ts index d2a1d36c18..7f0ff4be06 100644 --- a/packages/json-schema/test/utils.ts +++ b/packages/json-schema/test/utils.ts @@ -1,3 +1,4 @@ +import { Diagnostic } from "@typespec/compiler"; import { createAssetEmitter } from "@typespec/compiler/emitter-framework"; import { createTestHost } from "@typespec/compiler/testing"; import { parse } from "yaml"; @@ -25,6 +26,15 @@ export async function emitSchema( options: JSONSchemaEmitterOptions = {}, testOptions: { emitNamespace?: boolean; emitTypes?: string[] } = { emitNamespace: true } ) { + const [schemas, _] = await emitSchemaWithDiagnostics(code, options, testOptions); + return schemas; +} + +export async function emitSchemaWithDiagnostics( + code: string, + options: JSONSchemaEmitterOptions = {}, + testOptions: { emitNamespace?: boolean; emitTypes?: string[] } = { emitNamespace: true } +): Promise<[Record, readonly Diagnostic[]]> { if (!options["file-type"]) { options["file-type"] = "json"; } @@ -58,5 +68,5 @@ export async function emitSchema( } } - return schemas; + return [schemas, host.program.diagnostics]; } From a5720019f2bb22524d0a21b6b07e0e63f58cf754 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 3 Nov 2023 13:48:48 -0700 Subject: [PATCH 08/24] add tm language --- packages/compiler/src/server/tmlanguage.ts | 20 +++- .../compiler/test/server/colorization.test.ts | 103 ++++++++++++++++-- 2 files changed, 114 insertions(+), 9 deletions(-) diff --git a/packages/compiler/src/server/tmlanguage.ts b/packages/compiler/src/server/tmlanguage.ts index 0f5620a7c7..d94cc38313 100644 --- a/packages/compiler/src/server/tmlanguage.ts +++ b/packages/compiler/src/server/tmlanguage.ts @@ -44,6 +44,8 @@ export type TypeSpecScope = | "punctuation.terminator.statement.tsp" | "punctuation.definition.typeparameters.begin.tsp" | "punctuation.definition.typeparameters.end.tsp" + | "punctuation.definition.template-expression.begin.tsp" + | "punctuation.definition.template-expression.end.tsp" | "punctuation.squarebracket.open.tsp" | "punctuation.squarebracket.close.tsp" | "punctuation.curlybrace.open.tsp" @@ -109,12 +111,26 @@ const escapeChar: MatchRule = { match: "\\\\.", }; +const templateExpression: BeginEndRule = { + key: "template-expression", + scope: meta, + begin: "\\$\\{", + beginCaptures: { + "0": { scope: "punctuation.definition.template-expression.begin.tsp" }, + }, + end: "\\}", + endCaptures: { + "0": { scope: "punctuation.definition.template-expression.end.tsp" }, + }, + patterns: [expression], +}; + const stringLiteral: BeginEndRule = { key: "string-literal", scope: "string.quoted.double.tsp", begin: '"', end: '"|$', - patterns: [escapeChar], + patterns: [templateExpression, escapeChar], }; const tripleQuotedStringLiteral: BeginEndRule = { @@ -122,7 +138,7 @@ const tripleQuotedStringLiteral: BeginEndRule = { scope: "string.quoted.triple.tsp", begin: '"""', end: '"""', - patterns: [escapeChar], + patterns: [templateExpression, escapeChar], }; const punctuationComma: MatchRule = { diff --git a/packages/compiler/test/server/colorization.test.ts b/packages/compiler/test/server/colorization.test.ts index ae991dcce0..1b9204e993 100644 --- a/packages/compiler/test/server/colorization.test.ts +++ b/packages/compiler/test/server/colorization.test.ts @@ -97,12 +97,18 @@ const Token = { begin: createToken("<", "punctuation.definition.typeparameters.begin.tsp"), end: createToken(">", "punctuation.definition.typeparameters.end.tsp"), }, + templateExpression: { + begin: createToken("${", "punctuation.definition.template-expression.begin.tsp"), + end: createToken("}", "punctuation.definition.template-expression.end.tsp"), + }, }, literals: { + escape: (char: string) => createToken(`\\${char}`, "constant.character.escape.tsp"), numeric: (text: string) => createToken(text, "constant.numeric.tsp"), - string: (text: string) => - createToken(text.startsWith('"') ? text : '"' + text + '"', "string.quoted.double.tsp"), + stringQuoted: (text: string) => createToken('"' + text + '"', "string.quoted.double.tsp"), + string: (text: string) => createToken(text, "string.quoted.double.tsp"), + stringTriple: (text: string) => createToken(text, "string.quoted.triple.tsp"), }, comment: { block: (text: string) => createToken(text, "comment.block.tsp"), @@ -115,6 +121,89 @@ testColorization("tmlanguage", tokenizeTMLanguage); function testColorization(description: string, tokenize: Tokenize) { describe(`compiler: server: ${description}`, () => { + describe("strings", () => { + describe("single line", () => { + it("tokenize template", async () => { + const tokens = await tokenize(`"Start \${123} end"`); + deepStrictEqual(tokens, [ + Token.literals.string('"'), + Token.literals.string("Start "), + Token.punctuation.templateExpression.begin, + Token.literals.numeric("123"), + Token.punctuation.templateExpression.end, + Token.literals.string(" end"), + Token.literals.string('"'), + ]); + }); + + it("tokenize as a string if the template expression are escaped", async () => { + const tokens = await tokenize(`"Start \\\${123} end"`); + deepStrictEqual(tokens, [ + Token.literals.string('"'), + Token.literals.string("Start "), + Token.literals.escape("$"), + Token.literals.string("{123} end"), + Token.literals.string('"'), + ]); + }); + + it("tokenize as a string if it is a simple string", async () => { + const tokens = await tokenize(`"Start end"`); + deepStrictEqual(tokens, [Token.literals.stringQuoted("Start end")]); + }); + }); + + describe("multi line", () => { + it("tokenize template", async () => { + const tokens = await tokenize(`""" + Start \${123} + end + """`); + deepStrictEqual(tokens, [ + Token.literals.stringTriple('"""'), + Token.literals.stringTriple(" Start "), + Token.punctuation.templateExpression.begin, + Token.literals.numeric("123"), + Token.punctuation.templateExpression.end, + Token.literals.stringTriple(" "), + Token.literals.stringTriple(" end"), + Token.literals.stringTriple(" "), + Token.literals.stringTriple('"""'), + ]); + }); + + it("tokenize as a string if the template expression are escaped", async () => { + const tokens = await tokenize(`""" + Start \\\${123} + end + """`); + deepStrictEqual(tokens, [ + Token.literals.stringTriple('"""'), + Token.literals.stringTriple(" Start "), + Token.literals.escape("$"), + Token.literals.stringTriple("{123} "), + Token.literals.stringTriple(" end"), + Token.literals.stringTriple(" "), + Token.literals.stringTriple('"""'), + ]); + }); + + it("tokenize as a simple string", async () => { + const tokens = await tokenize(`""" + Start + end + """`); + deepStrictEqual(tokens, [ + Token.literals.stringTriple(`"""`), + Token.literals.stringTriple(" Start"), + Token.literals.stringTriple(" end"), + Token.literals.stringTriple(" "), + Token.literals.stringTriple(`"""`), + ]); + }); + }); + }); + describe("aliases", () => { it("simple alias", async () => { const tokens = await tokenize("alias Foo = string"); @@ -213,7 +302,7 @@ function testColorization(description: string, tokenize: Tokenize) { Token.identifiers.tag("@"), Token.identifiers.tag("foo"), Token.punctuation.openParen, - Token.literals.string("param1"), + Token.literals.stringQuoted("param1"), Token.punctuation.comma, Token.literals.numeric("123"), Token.punctuation.closeParen, @@ -226,7 +315,7 @@ function testColorization(description: string, tokenize: Tokenize) { Token.punctuation.openParen, Token.identifiers.type("MyModel"), Token.punctuation.comma, - Token.literals.string("param1"), + Token.literals.stringQuoted("param1"), Token.punctuation.comma, Token.literals.numeric("123"), Token.punctuation.closeParen, @@ -532,7 +621,7 @@ function testColorization(description: string, tokenize: Tokenize) { Token.operators.typeAnnotation, Token.identifiers.type("string"), Token.operators.assignment, - Token.literals.string("my-default"), + Token.literals.stringQuoted("my-default"), Token.punctuation.semicolon, Token.punctuation.closeBrace, ]); @@ -714,7 +803,7 @@ function testColorization(description: string, tokenize: Tokenize) { Token.operators.typeAnnotation, Token.identifiers.type("string"), Token.operators.assignment, - Token.literals.string("my-default"), + Token.literals.stringQuoted("my-default"), Token.punctuation.closeParen, Token.operators.typeAnnotation, @@ -1131,7 +1220,7 @@ export async function tokenizeSemantic(input: string): Promise { case SemanticTokenKind.Keyword: return Token.keywords.other(text); case SemanticTokenKind.String: - return Token.literals.string(text); + return Token.literals.stringQuoted(text); case SemanticTokenKind.Comment: return Token.comment.block(text); case SemanticTokenKind.Number: From 71741b113370f4d3c25d97fbc54d0941ae3b33bd Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 3 Nov 2023 15:20:16 -0700 Subject: [PATCH 09/24] Colorization --- packages/compiler/src/server/serverlib.ts | 107 +++++++++++++++- .../compiler/test/server/colorization.test.ts | 121 +++++++++++++----- 2 files changed, 188 insertions(+), 40 deletions(-) diff --git a/packages/compiler/src/server/serverlib.ts b/packages/compiler/src/server/serverlib.ts index 8b25ab671f..3609dfe425 100644 --- a/packages/compiler/src/server/serverlib.ts +++ b/packages/compiler/src/server/serverlib.ts @@ -937,17 +937,12 @@ export function createServer(host: ServerHost): Server { function mapTokens() { const tokens = new Map(); const scanner = createScanner(file, () => {}); - + const templateStack: [Token, TokenFlags][] = []; while (scanner.scan() !== Token.EndOfFile) { if (scanner.tokenFlags & TokenFlags.DocComment) { classifyDocComment({ pos: scanner.tokenPosition, end: scanner.position }); } else { - const kind = classifyToken(scanner.token); - if (kind === ignore) { - continue; - } - tokens.set(scanner.tokenPosition, { - kind: kind === defer ? undefined! : kind, + handleToken(scanner.token, scanner.tokenFlags, { pos: scanner.tokenPosition, end: scanner.position, }); @@ -970,6 +965,104 @@ export function createServer(host: ServerHost): Server { } }); } + + function handleToken(token: Token, tokenFlags: TokenFlags, range: TextRange) { + switch (token) { + case Token.StringTemplateHead: + templateStack.push([token, tokenFlags]); + classifyStringTemplate(token, range); + break; + case Token.OpenBrace: + // If we don't have anything on the template stack, + // then we aren't trying to keep track of a previously scanned template head. + if (templateStack.length > 0) { + templateStack.push([token, tokenFlags]); + } + handleSimpleToken(token, range); + break; + case Token.CloseBrace: + // If we don't have anything on the template stack, + // then we aren't trying to keep track of a previously scanned template head. + if (templateStack.length > 0) { + const [lastToken, lastTokenFlags] = templateStack[templateStack.length - 1]; + + if (lastToken === Token.StringTemplateHead) { + token = scanner.reScanStringTemplate(lastTokenFlags); + + // Only pop on a TemplateTail; a TemplateMiddle indicates there is more for us. + if (token === Token.StringTemplateTail) { + templateStack.pop(); + classifyStringTemplate(token, { + pos: scanner.tokenPosition, + end: scanner.position, + }); + } else { + compilerAssert( + token === Token.StringTemplateMiddle, + "Should have been a template middle." + ); + classifyStringTemplate(token, { + pos: scanner.tokenPosition, + end: scanner.position, + }); + } + } else { + compilerAssert(lastToken === Token.OpenBrace, "Should have been an open brace"); + templateStack.pop(); + } + break; + } + handleSimpleToken(token, range); + break; + default: + handleSimpleToken(token, range); + } + } + + function handleSimpleToken(token: Token, range: TextRange) { + const kind = classifyToken(scanner.token); + if (kind === ignore) { + return; + } + tokens.set(range.pos, { + kind: kind === defer ? undefined! : kind, + ...range, + }); + } + + function classifyStringTemplate( + token: Token.StringTemplateHead | Token.StringTemplateMiddle | Token.StringTemplateTail, + range: TextRange + ) { + switch (token) { + case Token.StringTemplateHead: + case Token.StringTemplateMiddle: + const exprBeginPunctuation = range.end - 2; + tokens.set(range.pos, { + kind: SemanticTokenKind.String, + pos: range.pos, + end: exprBeginPunctuation, + }); + tokens.set(exprBeginPunctuation, { + kind: SemanticTokenKind.Operator, + pos: exprBeginPunctuation, + end: range.end, + }); + break; + case Token.StringTemplateTail: + const exprEndPunctuation = range.pos + 1; + tokens.set(range.pos, { + kind: SemanticTokenKind.Operator, + pos: range.pos, + end: exprEndPunctuation, + }); + tokens.set(exprEndPunctuation, { + kind: SemanticTokenKind.String, + pos: exprEndPunctuation, + end: range.end, + }); + } + } } function classifyToken(token: Token): SemanticTokenKind | typeof defer | typeof ignore { diff --git a/packages/compiler/test/server/colorization.test.ts b/packages/compiler/test/server/colorization.test.ts index 1b9204e993..4f1841b9b4 100644 --- a/packages/compiler/test/server/colorization.test.ts +++ b/packages/compiler/test/server/colorization.test.ts @@ -120,30 +120,43 @@ testColorization("semantic colorization", tokenizeSemantic); testColorization("tmlanguage", tokenizeTMLanguage); function testColorization(description: string, tokenize: Tokenize) { + function joinTokensInSemantic(tokens: T[], separator: "" | "\n" = ""): T[] { + if (tokenize === tokenizeSemantic) { + return [createToken(tokens.map((x) => x.text).join(separator), tokens[0].scope)] as any; + } + return tokens; + } + describe(`compiler: server: ${description}`, () => { describe("strings", () => { + function templateTripleOrDouble(text: string): Token { + return tokenize === tokenizeSemantic + ? Token.literals.string(text) + : Token.literals.stringTriple(text); + } + describe("single line", () => { it("tokenize template", async () => { const tokens = await tokenize(`"Start \${123} end"`); deepStrictEqual(tokens, [ - Token.literals.string('"'), - Token.literals.string("Start "), + ...joinTokensInSemantic([Token.literals.string('"'), Token.literals.string("Start ")]), Token.punctuation.templateExpression.begin, Token.literals.numeric("123"), Token.punctuation.templateExpression.end, - Token.literals.string(" end"), - Token.literals.string('"'), + ...joinTokensInSemantic([Token.literals.string(" end"), Token.literals.string('"')]), ]); }); - + []; it("tokenize as a string if the template expression are escaped", async () => { const tokens = await tokenize(`"Start \\\${123} end"`); deepStrictEqual(tokens, [ - Token.literals.string('"'), - Token.literals.string("Start "), - Token.literals.escape("$"), - Token.literals.string("{123} end"), - Token.literals.string('"'), + ...joinTokensInSemantic([ + Token.literals.string('"'), + Token.literals.string("Start "), + Token.literals.escape("$"), + Token.literals.string("{123} end"), + Token.literals.string('"'), + ]), ]); }); @@ -160,15 +173,24 @@ function testColorization(description: string, tokenize: Tokenize) { end """`); deepStrictEqual(tokens, [ - Token.literals.stringTriple('"""'), - Token.literals.stringTriple(" Start "), + ...joinTokensInSemantic( + [Token.literals.stringTriple('"""'), Token.literals.stringTriple(" Start ")], + "\n" + ), Token.punctuation.templateExpression.begin, Token.literals.numeric("123"), Token.punctuation.templateExpression.end, - Token.literals.stringTriple(" "), - Token.literals.stringTriple(" end"), - Token.literals.stringTriple(" "), - Token.literals.stringTriple('"""'), + ...joinTokensInSemantic( + [ + templateTripleOrDouble(" "), + templateTripleOrDouble(" end"), + ...joinTokensInSemantic([ + templateTripleOrDouble(" "), + templateTripleOrDouble('"""'), + ]), + ], + "\n" + ), ]); }); @@ -178,13 +200,22 @@ function testColorization(description: string, tokenize: Tokenize) { end """`); deepStrictEqual(tokens, [ - Token.literals.stringTriple('"""'), - Token.literals.stringTriple(" Start "), - Token.literals.escape("$"), - Token.literals.stringTriple("{123} "), - Token.literals.stringTriple(" end"), - Token.literals.stringTriple(" "), - Token.literals.stringTriple('"""'), + ...joinTokensInSemantic( + [ + Token.literals.stringTriple('"""'), + ...joinTokensInSemantic([ + Token.literals.stringTriple(" Start "), + Token.literals.escape("$"), + Token.literals.stringTriple("{123} "), + ]), + Token.literals.stringTriple(" end"), + ...joinTokensInSemantic([ + Token.literals.stringTriple(" "), + Token.literals.stringTriple('"""'), + ]), + ], + "\n" + ), ]); }); @@ -194,11 +225,18 @@ function testColorization(description: string, tokenize: Tokenize) { end """`); deepStrictEqual(tokens, [ - Token.literals.stringTriple(`"""`), - Token.literals.stringTriple(" Start"), - Token.literals.stringTriple(" end"), - Token.literals.stringTriple(" "), - Token.literals.stringTriple(`"""`), + ...joinTokensInSemantic( + [ + Token.literals.stringTriple(`"""`), + Token.literals.stringTriple(" Start"), + Token.literals.stringTriple(" end"), + ...joinTokensInSemantic([ + Token.literals.stringTriple(" "), + Token.literals.stringTriple(`"""`), + ]), + ], + "\n" + ), ]); }); }); @@ -1188,11 +1226,24 @@ export async function tokenizeSemantic(input: string): Promise { const semanticTokens = await host.server.getSemanticTokens({ textDocument: document }); const tokens = []; + let templateStack = 0; for (const semanticToken of semanticTokens) { const text = file.text.substring(semanticToken.pos, semanticToken.end); - const token = convertSemanticToken(semanticToken, text); - if (token) { - tokens.push(token); + if (text === "${" && semanticToken.kind === SemanticTokenKind.Operator) { + templateStack++; + tokens.push(Token.punctuation.templateExpression.begin); + } else if ( + templateStack > 0 && + text === "}" && + semanticToken.kind === SemanticTokenKind.Operator + ) { + templateStack--; + tokens.push(Token.punctuation.templateExpression.end); + } else { + const token = convertSemanticToken(semanticToken, text); + if (token) { + tokens.push(token); + } } } @@ -1220,7 +1271,9 @@ export async function tokenizeSemantic(input: string): Promise { case SemanticTokenKind.Keyword: return Token.keywords.other(text); case SemanticTokenKind.String: - return Token.literals.stringQuoted(text); + return text.startsWith(`"""`) + ? Token.literals.stringTriple(text) + : Token.literals.string(text); case SemanticTokenKind.Comment: return Token.comment.block(text); case SemanticTokenKind.Number: @@ -1358,7 +1411,9 @@ function getPunctuationMap(): ReadonlyMap { if ("text" in value) { map.set(value.text, value); } else { - visit(value); + if (value !== Token.punctuation.templateExpression) { + visit(value); + } } } } From e027aa505501354c82d6ff07c7e85bf9c4066a85 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 3 Nov 2023 15:50:32 -0700 Subject: [PATCH 10/24] Fix issue with multiple segments --- packages/compiler/src/server/serverlib.ts | 48 ++++++++----------- .../compiler/test/server/colorization.test.ts | 17 ++++++- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/packages/compiler/src/server/serverlib.ts b/packages/compiler/src/server/serverlib.ts index 3609dfe425..14930247c0 100644 --- a/packages/compiler/src/server/serverlib.ts +++ b/packages/compiler/src/server/serverlib.ts @@ -1034,33 +1034,27 @@ export function createServer(host: ServerHost): Server { token: Token.StringTemplateHead | Token.StringTemplateMiddle | Token.StringTemplateTail, range: TextRange ) { - switch (token) { - case Token.StringTemplateHead: - case Token.StringTemplateMiddle: - const exprBeginPunctuation = range.end - 2; - tokens.set(range.pos, { - kind: SemanticTokenKind.String, - pos: range.pos, - end: exprBeginPunctuation, - }); - tokens.set(exprBeginPunctuation, { - kind: SemanticTokenKind.Operator, - pos: exprBeginPunctuation, - end: range.end, - }); - break; - case Token.StringTemplateTail: - const exprEndPunctuation = range.pos + 1; - tokens.set(range.pos, { - kind: SemanticTokenKind.Operator, - pos: range.pos, - end: exprEndPunctuation, - }); - tokens.set(exprEndPunctuation, { - kind: SemanticTokenKind.String, - pos: exprEndPunctuation, - end: range.end, - }); + const stringStart = token === Token.StringTemplateHead ? range.pos : range.pos + 1; + const stringEnd = token === Token.StringTemplateTail ? range.end : range.end - 2; + + if (stringStart !== range.pos) { + tokens.set(range.pos, { + kind: SemanticTokenKind.Operator, + pos: range.pos, + end: stringStart, + }); + } + tokens.set(stringStart, { + kind: SemanticTokenKind.String, + pos: stringStart, + end: stringEnd, + }); + if (stringEnd !== range.end) { + tokens.set(stringEnd, { + kind: SemanticTokenKind.Operator, + pos: stringEnd, + end: range.end, + }); } } } diff --git a/packages/compiler/test/server/colorization.test.ts b/packages/compiler/test/server/colorization.test.ts index 4f1841b9b4..a7a74d65a6 100644 --- a/packages/compiler/test/server/colorization.test.ts +++ b/packages/compiler/test/server/colorization.test.ts @@ -146,7 +146,22 @@ function testColorization(description: string, tokenize: Tokenize) { ...joinTokensInSemantic([Token.literals.string(" end"), Token.literals.string('"')]), ]); }); - []; + + it("tokenize template with multiple interpolation", async () => { + const tokens = await tokenize(`"Start \${123} middle \${456} end"`); + deepStrictEqual(tokens, [ + ...joinTokensInSemantic([Token.literals.string('"'), Token.literals.string("Start ")]), + Token.punctuation.templateExpression.begin, + Token.literals.numeric("123"), + Token.punctuation.templateExpression.end, + Token.literals.string(" middle "), + Token.punctuation.templateExpression.begin, + Token.literals.numeric("456"), + Token.punctuation.templateExpression.end, + ...joinTokensInSemantic([Token.literals.string(" end"), Token.literals.string('"')]), + ]); + }); + it("tokenize as a string if the template expression are escaped", async () => { const tokens = await tokenize(`"Start \\\${123} end"`); deepStrictEqual(tokens, [ From dad6fd703dec70cacd4afdc177d2cb3c45f81fcc Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Mon, 6 Nov 2023 08:39:41 -0800 Subject: [PATCH 11/24] Add parsing test for model expression in string template --- packages/compiler/test/parser.test.ts | 14 ++++++++++++++ packages/compiler/test/scanner.test.ts | 24 ++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/packages/compiler/test/parser.test.ts b/packages/compiler/test/parser.test.ts index 8392663bbb..43f26a63ef 100644 --- a/packages/compiler/test/parser.test.ts +++ b/packages/compiler/test/parser.test.ts @@ -553,6 +553,20 @@ describe("compiler: parser", () => { strictEqual(span1.expression.value, 23); }); + it("parse a single line template with a multi line model expression inside", () => { + const astNode = parseSuccessWithLog( + `alias T = "Start \${{ foo: "one",\nbar: "two" }} end";` + ); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.value, "Start "); + strictEqual(node.spans.length, 1); + + const span0 = node.spans[0]; + strictEqual(span0.literal.value, " end"); + strictEqual(span0.expression.kind, SyntaxKind.ModelExpression); + strictEqual(span0.expression.properties.length, 2); + }); + it("can escape some ${}", () => { const astNode = parseSuccessWithLog(`alias T = "Start \${"one"} middle \\\${23} end";`); const node = getStringTemplateNode(astNode); diff --git a/packages/compiler/test/scanner.test.ts b/packages/compiler/test/scanner.test.ts index 5cb7be94fc..8b52471635 100644 --- a/packages/compiler/test/scanner.test.ts +++ b/packages/compiler/test/scanner.test.ts @@ -13,6 +13,8 @@ import { isPunctuation, isStatementKeyword, } from "../src/core/scanner.js"; +import { DiagnosticMatch, expectDiagnostics } from "../src/testing/expect.js"; +import { extractSquiggles } from "../src/testing/test-server-host.js"; type TokenEntry = [ Token, @@ -215,10 +217,18 @@ describe("compiler: scanner", () => { ]); }); - function scanString(text: string, expectedValue: string, expectedDiagnostic?: RegExp) { + function scanString( + text: string, + expectedValue: string, + expectedDiagnostic?: RegExp | DiagnosticMatch + ) { const scanner = createScanner(text, (diagnostic) => { if (expectedDiagnostic) { - assert.match(diagnostic.message, expectedDiagnostic); + if (expectedDiagnostic instanceof RegExp) { + assert.match(diagnostic.message, expectedDiagnostic); + } else { + expectDiagnostics([diagnostic], expectedDiagnostic); + } } else { assert.fail("No diagnostic expected, but got " + formatDiagnostic(diagnostic)); } @@ -240,6 +250,16 @@ describe("compiler: scanner", () => { scanString('"Hello world \\r\\n \\t \\" \\\\ !"', 'Hello world \r\n \t " \\ !'); }); + it("report diagnostic when escaping invalid char", () => { + const { source, pos, end } = extractSquiggles('"Hello world ~~~\\d~~~"'); + scanString(source, "Hello world d", { + code: "invalid-escape-sequence", + message: "Invalid escape sequence.", + pos, + end, + }); + }); + it("does not allow multi-line, non-triple-quoted strings", () => { scanString('"More\r\nthan\r\none\r\nline"', "More", /Unterminated string/); scanString('"More\nthan\none\nline"', "More", /Unterminated string/); From ad23a852e5c12e441b471c48481a52d0a9a51513 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Mon, 6 Nov 2023 09:25:24 -0800 Subject: [PATCH 12/24] Format --- .../compiler/src/formatter/print/printer.ts | 24 +++++++++ .../compiler/test/formatter/formatter.test.ts | 50 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/packages/compiler/src/formatter/print/printer.ts b/packages/compiler/src/formatter/print/printer.ts index 85f75c91f7..a2f4a1fc2a 100644 --- a/packages/compiler/src/formatter/print/printer.ts +++ b/packages/compiler/src/formatter/print/printer.ts @@ -60,6 +60,8 @@ import { ScalarStatementNode, Statement, StringLiteralNode, + StringTemplateExpressionNode, + StringTemplateSpanNode, SyntaxKind, TemplateParameterDeclarationNode, TextRange, @@ -358,6 +360,11 @@ export function printNode( case SyntaxKind.EmptyStatement: return ""; case SyntaxKind.StringTemplateExpression: + return printStringTemplateExpression( + path as AstPath, + options, + print + ); case SyntaxKind.StringTemplateSpan: case SyntaxKind.StringTemplateHead: case SyntaxKind.StringTemplateMiddle: @@ -1677,6 +1684,23 @@ export function printReturnExpression( return ["return ", path.call(print, "value")]; } +export function printStringTemplateExpression( + path: AstPath, + options: TypeSpecPrettierOptions, + print: PrettierChildPrint +) { + const node = path.node; + const headText = getRawText(node.head, options); + const content = [ + getRawText(node.head, options), + path.map((span: AstPath) => { + const expression = span.call(print, "expression"); + return [expression, getRawText(span.node.literal, options)]; + }, "spans"), + ]; + return content; +} + function printItemList( path: AstPath, options: TypeSpecPrettierOptions, diff --git a/packages/compiler/test/formatter/formatter.test.ts b/packages/compiler/test/formatter/formatter.test.ts index 5c7f9faf0b..ccb906490b 100644 --- a/packages/compiler/test/formatter/formatter.test.ts +++ b/packages/compiler/test/formatter/formatter.test.ts @@ -2693,4 +2693,54 @@ op test(): string; }); }); }); + + describe("string templates", () => { + describe("sginle line", () => { + it("format simple single line string template", async () => { + await assertFormat({ + code: `alias T = "foo \${ "def" } baz";`, + expected: `alias T = "foo \${"def"} baz";`, + }); + }); + + it("format simple single line string template with multiple interpolation", async () => { + await assertFormat({ + code: `alias T = "foo \${ "one" } bar \${"two" } baz";`, + expected: `alias T = "foo \${"one"} bar \${"two"} baz";`, + }); + }); + + it("format model expression in single line string template", async () => { + await assertFormat({ + code: `alias T = "foo \${ {foo: 1, bar: 2} } baz";`, + expected: ` +alias T = "foo \${{ + foo: 1; + bar: 2; +}} baz"; + `, + }); + }); + }); + describe("triple quoted", () => { + it("format simple single line string template", async () => { + await assertFormat({ + code: ` +alias T = """ + This \${ "one" } goes over + multiple + \${ "two" } + lines + """;`, + expected: ` +alias T = """ + This \${"one"} goes over + multiple + \${"two"} + lines + """;`, + }); + }); + }); + }); }); From a84c0a60a813bd3490fdfdd1c80e40b1f9e20be7 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Mon, 6 Nov 2023 09:28:06 -0800 Subject: [PATCH 13/24] . --- packages/compiler/src/formatter/print/printer.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/compiler/src/formatter/print/printer.ts b/packages/compiler/src/formatter/print/printer.ts index a2f4a1fc2a..9949641632 100644 --- a/packages/compiler/src/formatter/print/printer.ts +++ b/packages/compiler/src/formatter/print/printer.ts @@ -1690,7 +1690,6 @@ export function printStringTemplateExpression( print: PrettierChildPrint ) { const node = path.node; - const headText = getRawText(node.head, options); const content = [ getRawText(node.head, options), path.map((span: AstPath) => { From f98ab5228ac078178ce7c1ed9d68181aa7c42193 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Mon, 13 Nov 2023 15:15:53 -0800 Subject: [PATCH 14/24] update grammar --- .../compiler/test/formatter/formatter.test.ts | 2 +- packages/spec/src/spec.emu.html | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/compiler/test/formatter/formatter.test.ts b/packages/compiler/test/formatter/formatter.test.ts index ccb906490b..ecc69101a5 100644 --- a/packages/compiler/test/formatter/formatter.test.ts +++ b/packages/compiler/test/formatter/formatter.test.ts @@ -2695,7 +2695,7 @@ op test(): string; }); describe("string templates", () => { - describe("sginle line", () => { + describe("single line", () => { it("format simple single line string template", async () => { await assertFormat({ code: `alias T = "foo \${ "def" } baz";`, diff --git a/packages/spec/src/spec.emu.html b/packages/spec/src/spec.emu.html index 144a433acc..1420c682f7 100644 --- a/packages/spec/src/spec.emu.html +++ b/packages/spec/src/spec.emu.html @@ -154,14 +154,28 @@

Lexical Grammar

StringLiteral : `"` StringCharacters? `"` `"""` TripleQuotedStringCharacters? `"""` + StringTemplateHead + +StringTemplateHead :: + `"` StringCharacters? `${` + `"""` TripleQuotedStringCharacters? `${` + +StringTemplateMiddle :: + `}` TemplateCharacters? `${` + + +StringTemplateTail :: + `}` TemplateCharacters? `"` + `}` TemplateCharacters? `"""` StringCharacters : StringCharacter StringCharacters? StringCharacter : + `$` [lookahead != `{`] SourceCharacter but not one of `"` or `\` or LineTerminator `\` EscapeCharacter - + /// // BUG: This does not specify the extra rules about `"""`s going // on their own lines and having consistent indentation. From 16e7f8b4ff83209c713ef294e7143ec22809d646 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Mon, 13 Nov 2023 15:55:24 -0800 Subject: [PATCH 15/24] Auto marshaling of values --- packages/compiler/src/core/checker.ts | 16 +++++- .../src/core/helpers/type-name-utils.ts | 2 + .../compiler/test/checker/decorators.test.ts | 54 +++++++++++++++++++ .../compiler/test/checker/relation.test.ts | 4 ++ 4 files changed, 75 insertions(+), 1 deletion(-) diff --git a/packages/compiler/src/core/checker.ts b/packages/compiler/src/core/checker.ts index b682e789cd..8943f1a861 100644 --- a/packages/compiler/src/core/checker.ts +++ b/packages/compiler/src/core/checker.ts @@ -3,7 +3,12 @@ import { createSymbol, createSymbolTable } from "./binder.js"; import { getDeprecationDetails, markDeprecated } from "./deprecation.js"; import { ProjectionError, compilerAssert, reportDeprecated } from "./diagnostics.js"; import { validateInheritanceDiscriminatedUnions } from "./helpers/discriminator-utils.js"; -import { TypeNameOptions, getNamespaceFullName, getTypeName } from "./helpers/index.js"; +import { + TypeNameOptions, + getNamespaceFullName, + getTypeName, + stringTemplateToString, +} from "./helpers/index.js"; import { createDiagnostic } from "./messages.js"; import { getIdentifierContext, hasParseError, visitChildren } from "./parser.js"; import { Program, ProjectedProgram } from "./program.js"; @@ -3295,6 +3300,10 @@ export function createChecker(program: Program): Checker { if (type === nullType) { return true; } + if (type.kind === "StringTemplate") { + const [_, diagnostics] = stringTemplateToString(type); + return diagnostics.length === 0; + } const valueTypes = new Set(["String", "Number", "Boolean", "EnumMember", "Tuple"]); return valueTypes.has(type.kind); } @@ -3476,6 +3485,8 @@ export function createChecker(program: Program): Checker { if (valueOf) { if (value.kind === "Boolean" || value.kind === "String" || value.kind === "Number") { return literalTypeToValue(value); + } else if (value.kind === "StringTemplate") { + return stringTemplateToString(value)[0]; } } return value; @@ -5332,6 +5343,7 @@ export function createChecker(program: Program): Checker { case "Number": return isNumericLiteralRelatedTo(source, target); case "String": + case "StringTemplate": return areScalarsRelated(target, getStdType("string")); case "Boolean": return areScalarsRelated(target, getStdType("boolean")); @@ -6105,6 +6117,8 @@ function marshalArgumentsForJS(args: T[]): MarshalledValue[] return args.map((arg) => { if (arg.kind === "Boolean" || arg.kind === "String" || arg.kind === "Number") { return literalTypeToValue(arg); + } else if (arg.kind === "StringTemplate") { + return stringTemplateToString(arg)[0]; } return arg as any; }); diff --git a/packages/compiler/src/core/helpers/type-name-utils.ts b/packages/compiler/src/core/helpers/type-name-utils.ts index e87aef8efb..32ea713ed9 100644 --- a/packages/compiler/src/core/helpers/type-name-utils.ts +++ b/packages/compiler/src/core/helpers/type-name-utils.ts @@ -45,6 +45,8 @@ export function getTypeName(type: Type | ValueType, options?: TypeNameOptions): return getTypeName(type.type, options); case "Tuple": return "[" + type.values.map((x) => getTypeName(x, options)).join(", ") + "]"; + case "StringTemplate": + return "string"; case "String": case "Number": case "Boolean": diff --git a/packages/compiler/test/checker/decorators.test.ts b/packages/compiler/test/checker/decorators.test.ts index 147d9cc458..ce3a0c6e24 100644 --- a/packages/compiler/test/checker/decorators.test.ts +++ b/packages/compiler/test/checker/decorators.test.ts @@ -273,6 +273,60 @@ describe("compiler: checker: decorators", () => { }, ]); }); + + describe("value marshalling", () => { + async function testCallDecorator(type: string, value: string): Promise { + await runner.compile(` + extern dec testDec(target: unknown, arg1: ${type}); + + @testDec(${value}) + @test + model Foo {} + `); + return calledArgs[2]; + } + + describe("passing a string literal", () => { + it("`: valueof string` cast the value to a JS string", async () => { + const arg = await testCallDecorator("valueof string", `"one"`); + strictEqual(arg, "one"); + }); + + it("`: string` keeps the StringLiteral type", async () => { + const arg = await testCallDecorator("string", `"one"`); + strictEqual(arg.kind, "String"); + }); + }); + + describe("passing a string template", () => { + it("`: valueof string` cast the value to a JS string", async () => { + const arg = await testCallDecorator( + "valueof string", + '"Start ${"one"} middle ${"two"} end"' + ); + strictEqual(arg, "Start one middle two end"); + }); + + it("`: string` keeps the StringTemplate type", async () => { + const arg = await testCallDecorator("string", '"Start ${"one"} middle ${"two"} end"'); + strictEqual(arg.kind, "StringTemplate"); + }); + }); + + describe("passing a numeric literal", () => { + it("valueof int32 cast the value to a JS number", async () => { + const arg = await testCallDecorator("valueof int32", `123`); + strictEqual(arg, 123); + }); + }); + + describe("passing a boolean literal", () => { + it("valueof boolean cast the value to a JS boolean", async () => { + const arg = await testCallDecorator("valueof boolean", `true`); + strictEqual(arg, true); + }); + }); + }); }); it("can have the same name as types", async () => { diff --git a/packages/compiler/test/checker/relation.test.ts b/packages/compiler/test/checker/relation.test.ts index 34e442c60d..a0cdd4d2a5 100644 --- a/packages/compiler/test/checker/relation.test.ts +++ b/packages/compiler/test/checker/relation.test.ts @@ -209,6 +209,10 @@ describe("compiler: checker: type relations", () => { await expectTypeAssignable({ source: `"foo"`, target: "string" }); }); + it("can assign string template with primitives interpolated", async () => { + await expectTypeAssignable({ source: `"foo \${123} bar"`, target: "string" }); + }); + it("can assign string literal union", async () => { await expectTypeAssignable({ source: `"foo" | "bar"`, target: "string" }); }); From a4b6d9ce0bbd6fe06559a5dc98673446c8ab9742 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Mon, 13 Nov 2023 16:00:55 -0800 Subject: [PATCH 16/24] Add string template support --- .../feature-string-template_2023-11-14-00-00.json | 10 ++++++++++ .../feature-string-template_2023-11-14-00-00.json | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 common/changes/@typespec/compiler/feature-string-template_2023-11-14-00-00.json create mode 100644 common/changes/@typespec/json-schema/feature-string-template_2023-11-14-00-00.json diff --git a/common/changes/@typespec/compiler/feature-string-template_2023-11-14-00-00.json b/common/changes/@typespec/compiler/feature-string-template_2023-11-14-00-00.json new file mode 100644 index 0000000000..fee8c10c10 --- /dev/null +++ b/common/changes/@typespec/compiler/feature-string-template_2023-11-14-00-00.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@typespec/compiler", + "comment": "**New language feature** **BREAKING** Added string template literal in typespec. Singel and multi-line strings can be interpolated with `${` and `}`. Example `\\`Doc for url ${url} is here: ${location}\\``", + "type": "none" + } + ], + "packageName": "@typespec/compiler" +} \ No newline at end of file diff --git a/common/changes/@typespec/json-schema/feature-string-template_2023-11-14-00-00.json b/common/changes/@typespec/json-schema/feature-string-template_2023-11-14-00-00.json new file mode 100644 index 0000000000..8943b687d9 --- /dev/null +++ b/common/changes/@typespec/json-schema/feature-string-template_2023-11-14-00-00.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@typespec/json-schema", + "comment": "Added support for string template literals", + "type": "none" + } + ], + "packageName": "@typespec/json-schema" +} \ No newline at end of file From 45c8a054f30e4dd086469bf7fc55bfec9c1455a0 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Mon, 13 Nov 2023 19:14:45 -0800 Subject: [PATCH 17/24] Docs and prism-js --- docs/language-basics/type-literals.md | 16 +++++++++ packages/website/docusaurus.config.ts | 2 +- .../website/src/theme/typespec-lang-prism.ts | 35 +++++++++++++++---- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/docs/language-basics/type-literals.md b/docs/language-basics/type-literals.md index cf6ee9af22..0a5706ccd3 100644 --- a/docs/language-basics/type-literals.md +++ b/docs/language-basics/type-literals.md @@ -62,6 +62,22 @@ two } ``` +## String template literal + +Single or multi line string literal can be interpolated using `${}` + +```typespec +alias hello = "bonjour"; +alias Single = "${hello} world!"; + +alias Multi = """ + ${hello} + world! + """; +``` + +Any valid expression can be used in the interpolation but only other literals will result in the template literal being assignable to a `valueof string`. Any other value will be dependent on the decorator/emitter receiving it to handle. + ## Numeric literal Numeric literals can be declared by using the raw number diff --git a/packages/website/docusaurus.config.ts b/packages/website/docusaurus.config.ts index 18dd06ffed..3de559be25 100644 --- a/packages/website/docusaurus.config.ts +++ b/packages/website/docusaurus.config.ts @@ -222,7 +222,7 @@ const config: Config = { }, prism: { theme: themes.oneLight, - darkTheme: themes.dracula, + darkTheme: themes.oneDark, additionalLanguages: [], }, mermaid: {}, diff --git a/packages/website/src/theme/typespec-lang-prism.ts b/packages/website/src/theme/typespec-lang-prism.ts index 9142a0c139..f47d7acfbf 100644 --- a/packages/website/src/theme/typespec-lang-prism.ts +++ b/packages/website/src/theme/typespec-lang-prism.ts @@ -1,4 +1,4 @@ -export default { +const lang = { comment: [ { // multiline comments eg /* ASDF */ @@ -34,14 +34,33 @@ export default { }, string: [ + // https://docs.swift.org/swift-book/LanguageGuide/StringsAndCharacters.html { - pattern: /"""[^"][\s\S]*?"""/, - greedy: true, - }, - { - pattern: /(^|[^\\"])"(?:\\.|\$(?!\{)|[^"\\\r\n$])*"/, + pattern: new RegExp( + /(^|[^"#])/.source + + "(?:" + + // multi-line string + /"""(?:\\(?:\$\{(?:[^{}]|\$\{[^{}]*\})*\}|[^(])|[^\\"]|"(?!""))*"""/.source + + "|" + + // single-line string + /"(?:\\(?:\$\{(?:[^{}]|\$\{[^{}]*\})*\}|\r\n|[^(])|[^\\\r\n"])*"/.source + + ")" + ), lookbehind: true, greedy: true, + inside: { + interpolation: { + pattern: /(\$\{)(?:[^{}]|\$\{[^{}]*\})*(?=\})/, + lookbehind: true, + inside: null, // see below + }, + "interpolation-punctuation": { + pattern: /^\}|\$\{$/, + alias: "punctuation", + }, + punctuation: /\\(?=[\r\n])/, + string: /[\s\S]+/, + }, }, ], @@ -50,9 +69,13 @@ export default { /\b(?:import|model|scalar|namespace|op|interface|union|using|is|extends|enum|alias|return|void|never|if|else|projection|dec|extern|fn)\b/, function: /\b[a-z_]\w*(?=[ \t]*\()/i, + variable: /\b(?:[A-Z_\d]*[a-z]\w*)?\b/, number: /(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:E[+-]?\d+)?/i, operator: /--|\+\+|\*\*=?|=>|&&=?|\|\|=?|[!=]==|<<=?|>>>?=?|[-+*/%&|^!=<>]=?|\.{3}|\?\?=?|\?\.?|[~:]/, punctuation: /[{}[\];(),.:]/, }; + +lang.string[0].inside.interpolation.inside = lang; +export default lang; From b0101fe93991310ccbead26680a462f0a6a311bb Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 16 Nov 2023 11:59:05 -0800 Subject: [PATCH 18/24] Update grammar --- packages/spec/src/spec.emu.html | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/spec/src/spec.emu.html b/packages/spec/src/spec.emu.html index 1420c682f7..2e9e081ea0 100644 --- a/packages/spec/src/spec.emu.html +++ b/packages/spec/src/spec.emu.html @@ -156,15 +156,26 @@

Lexical Grammar

`"""` TripleQuotedStringCharacters? `"""` StringTemplateHead -StringTemplateHead :: +StringTemplate : + TemplateHead Expression TemplateSpans + +TemplateSpans : + TemplateTail + TemplateMiddleList TemplateTail + +TemplateMiddleList : + StringTemplateMiddle Expression + StringTemplateMiddle Expression TemplateMiddleList + +StringTemplateHead : `"` StringCharacters? `${` `"""` TripleQuotedStringCharacters? `${` -StringTemplateMiddle :: +StringTemplateMiddle : `}` TemplateCharacters? `${` -StringTemplateTail :: +StringTemplateTail : `}` TemplateCharacters? `"` `}` TemplateCharacters? `"""` From f66688b0cf4818e82236703ec7f7615bfe330157 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 16 Nov 2023 11:59:41 -0800 Subject: [PATCH 19/24] Update common/changes/@typespec/compiler/feature-string-template_2023-11-14-00-00.json Co-authored-by: Brian Terlson --- .../compiler/feature-string-template_2023-11-14-00-00.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/changes/@typespec/compiler/feature-string-template_2023-11-14-00-00.json b/common/changes/@typespec/compiler/feature-string-template_2023-11-14-00-00.json index fee8c10c10..b1f017e2fc 100644 --- a/common/changes/@typespec/compiler/feature-string-template_2023-11-14-00-00.json +++ b/common/changes/@typespec/compiler/feature-string-template_2023-11-14-00-00.json @@ -2,7 +2,7 @@ "changes": [ { "packageName": "@typespec/compiler", - "comment": "**New language feature** **BREAKING** Added string template literal in typespec. Singel and multi-line strings can be interpolated with `${` and `}`. Example `\\`Doc for url ${url} is here: ${location}\\``", + "comment": "**New language feature** **BREAKING** Added string template literal in typespec. Single and multi-line strings can be interpolated with `${` and `}`. Example `\\`Doc for url ${url} is here: ${location}\\``", "type": "none" } ], From a8ace5c5d9c0fbbb200ac2c89b7fcb671597f7dc Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 16 Nov 2023 12:13:51 -0800 Subject: [PATCH 20/24] Add test --- .../src/core/helpers/string-template-utils.ts | 9 ++++++--- packages/compiler/test/parser.test.ts | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/packages/compiler/src/core/helpers/string-template-utils.ts b/packages/compiler/src/core/helpers/string-template-utils.ts index 366f439ac8..e6d1198d15 100644 --- a/packages/compiler/src/core/helpers/string-template-utils.ts +++ b/packages/compiler/src/core/helpers/string-template-utils.ts @@ -1,3 +1,4 @@ +import { createDiagnosticCollector } from "../index.js"; import { createDiagnostic } from "../messages.js"; import { Diagnostic, StringTemplate } from "../types.js"; import { getTypeName } from "./type-name-utils.js"; @@ -12,7 +13,7 @@ import { getTypeName } from "./type-name-utils.js"; export function stringTemplateToString( stringTemplate: StringTemplate ): [string, readonly Diagnostic[]] { - const diagnostics: Diagnostic[] = []; + const diagnostics = createDiagnosticCollector(); const result = stringTemplate.spans .map((x) => { if (x.isInterpolated) { @@ -21,8 +22,10 @@ export function stringTemplateToString( case "Number": case "Boolean": return String(x.type.value); + case "StringTemplate": + return diagnostics.pipe(stringTemplateToString(x.type)); default: - diagnostics.push( + diagnostics.add( createDiagnostic({ code: "non-literal-string-template", target: x.node, @@ -35,5 +38,5 @@ export function stringTemplateToString( } }) .join(""); - return [result, diagnostics]; + return diagnostics.wrap(result); } diff --git a/packages/compiler/test/parser.test.ts b/packages/compiler/test/parser.test.ts index 43f26a63ef..2328da829d 100644 --- a/packages/compiler/test/parser.test.ts +++ b/packages/compiler/test/parser.test.ts @@ -579,6 +579,26 @@ describe("compiler: parser", () => { strictEqual(span0.expression.value, "one"); }); + it("can nest string templates", () => { + const astNode = parseSuccessWithLog( + 'alias T = "Start ${"nested-start ${"hi"} nested-end"} end";' + ); + const node = getStringTemplateNode(astNode); + strictEqual(node.head.value, "Start "); + strictEqual(node.spans.length, 1); + + const span0 = node.spans[0]; + strictEqual(span0.literal.value, " end"); + strictEqual(span0.expression.kind, SyntaxKind.StringTemplateExpression); + strictEqual(span0.expression.head.value, "nested-start "); + strictEqual(span0.expression.spans.length, 1); + + const nestedSpan0 = span0.expression.spans[0]; + strictEqual(nestedSpan0.literal.value, " nested-end"); + strictEqual(nestedSpan0.expression.kind, SyntaxKind.StringLiteral); + strictEqual(nestedSpan0.expression.value, "hi"); + }); + it("string with all ${} escape is still a StringLiteral", () => { const astNode = parseSuccessWithLog(`alias T = "Start \\\${12} middle \\\${23} end";`); const node = getNode(astNode); From f103e5ab5712e0b50b9f65201747085f77c98891 Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 16 Nov 2023 13:00:54 -0800 Subject: [PATCH 21/24] Add test and docs on decorators --- docs/extending-typespec/create-decorators.md | 33 ++++++++++++ .../helpers/string-template-utils.test.ts | 50 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 packages/compiler/test/helpers/string-template-utils.test.ts diff --git a/docs/extending-typespec/create-decorators.md b/docs/extending-typespec/create-decorators.md index ac0727e597..a8bb3da727 100644 --- a/docs/extending-typespec/create-decorators.md +++ b/docs/extending-typespec/create-decorators.md @@ -124,6 +124,8 @@ For certain TypeSpec types(Literal types) the decorator do not receive the actua for all the other types they are not transformed. +Example: + ```ts export function $tag( context: DecoratorContext, @@ -133,6 +135,37 @@ export function $tag( ) {} ``` +#### String templates and marshalling + +If a decorator parameter type is `valueof string`, a string template passed to it will also be marshalled as a string. +The TypeSpec type system will already validate the string template can be serialized as a string. + +```tsp +extern dec doc(target: unknown, name: valueof string); + + +alias world = "world!"; +@doc("Hello ${world} ") // receive: "Hello world!" +@doc("Hello ${123} ") // receive: "Hello 123" +@doc("Hello ${true} ") // receive: "Hello true" + +model Bar {} +@doc("Hello ${Bar} ") // not called error + ^ String template cannot be serialized as a string. + +``` + +#### Typescript type Reference + +| TypeSpec Parameter Type | TypeScript types | +| ---------------------------- | -------------------------------------------- | +| `valueof string` | `string` | +| `valueof numeric` | `number` | +| `valueof boolean` | `boolean` | +| `string` | `StringLiteral \| TemplateLiteral \| Scalar` | +| `Reflection.StringLiteral` | `StringLiteral` | +| `Reflection.TemplateLiteral` | `TemplateLiteral` | + ### Adding metadata with decorators Decorators can be used to register some metadata. For this you can use the `context.program.stateMap` or `context.program.stateSet` to insert data that will be tied to the current execution. diff --git a/packages/compiler/test/helpers/string-template-utils.test.ts b/packages/compiler/test/helpers/string-template-utils.test.ts new file mode 100644 index 0000000000..1c2b2ea29e --- /dev/null +++ b/packages/compiler/test/helpers/string-template-utils.test.ts @@ -0,0 +1,50 @@ +import { strictEqual } from "assert"; +import { ModelProperty, stringTemplateToString } from "../../src/index.js"; +import { expectDiagnosticEmpty } from "../../src/testing/expect.js"; +import { createTestRunner } from "../../src/testing/test-host.js"; + +describe("compiler: stringTemplateToString", () => { + async function stringifyTemplate(template: string) { + const runner = await createTestRunner(); + const { value } = (await runner.compile(`model Foo { @test value: ${template}; }`)) as { + value: ModelProperty; + }; + + strictEqual(value.type.kind, "StringTemplate"); + return stringTemplateToString(value.type); + } + + async function expectTemplateToString(template: string, expectation: string) { + const [result, diagnostics] = await stringifyTemplate(template); + expectDiagnosticEmpty(diagnostics); + strictEqual(result, expectation); + } + + describe("interpolate types", () => { + it("string literal", async () => { + await expectTemplateToString('"Start ${"one"} end"', "Start one end"); + }); + + it("numeric literal", async () => { + await expectTemplateToString('"Start ${123} end"', "Start 123 end"); + }); + + it("boolean literal", async () => { + await expectTemplateToString('"Start ${true} end"', "Start true end"); + }); + + it("nested string template", async () => { + await expectTemplateToString( + '"Start ${"Nested-start ${"one"} nested-end"} end"', + "Start Nested-start one nested-end end" + ); + }); + }); + + it("stringify template with multiple spans", async () => { + await expectTemplateToString( + '"Start ${"one"} middle ${"two"} end"', + "Start one middle two end" + ); + }); +}); From 9867f86b58ac80931f111da97b59e5a0552f4cdd Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 16 Nov 2023 14:32:55 -0800 Subject: [PATCH 22/24] Fix grammar, add support for openapi3 --- .../src/emitter-framework/type-emitter.ts | 2 +- .../json-schema/test/string-template.test.ts | 2 +- packages/openapi3/src/schema-emitter.ts | 13 ++++ .../openapi3/test/string-template.test.ts | 66 +++++++++++++++++++ packages/openapi3/test/test-host.ts | 23 ++++++- packages/spec/src/spec.emu.html | 2 +- 6 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 packages/openapi3/test/string-template.test.ts diff --git a/packages/compiler/src/emitter-framework/type-emitter.ts b/packages/compiler/src/emitter-framework/type-emitter.ts index 579dbcf409..404a9ccfc0 100644 --- a/packages/compiler/src/emitter-framework/type-emitter.ts +++ b/packages/compiler/src/emitter-framework/type-emitter.ts @@ -465,7 +465,7 @@ export class TypeEmitter> { return {}; } - stringTemplate(string: StringTemplate): EmitterOutput { + stringTemplate(stringTemplate: StringTemplate): EmitterOutput { return this.emitter.result.none(); } diff --git a/packages/json-schema/test/string-template.test.ts b/packages/json-schema/test/string-template.test.ts index b221e9f848..5738116e05 100644 --- a/packages/json-schema/test/string-template.test.ts +++ b/packages/json-schema/test/string-template.test.ts @@ -2,7 +2,7 @@ import { expectDiagnostics } from "@typespec/compiler/testing"; import { deepStrictEqual } from "assert"; import { emitSchema, emitSchemaWithDiagnostics } from "./utils.js"; -describe("string templates", () => { +describe("json-schema: string templates", () => { describe("handle interpolating literals", () => { it("string", async () => { const schemas = await emitSchema(` diff --git a/packages/openapi3/src/schema-emitter.ts b/packages/openapi3/src/schema-emitter.ts index d7f86ce9f9..d9449bda8a 100644 --- a/packages/openapi3/src/schema-emitter.ts +++ b/packages/openapi3/src/schema-emitter.ts @@ -37,6 +37,8 @@ import { Program, Scalar, StringLiteral, + StringTemplate, + stringTemplateToString, Tuple, Type, TypeNameOptions, @@ -332,6 +334,17 @@ export class OpenAPI3SchemaEmitter extends TypeEmitter< return { type: "string", enum: [string.value] }; } + stringTemplate(string: StringTemplate): EmitterOutput { + const [value, diagnostics] = stringTemplateToString(string); + if (diagnostics.length > 0) { + this.emitter + .getProgram() + .reportDiagnostics(diagnostics.map((x) => ({ ...x, severity: "warning" }))); + return { type: "string" }; + } + return { type: "string", enum: [value] }; + } + numericLiteral(number: NumericLiteral): EmitterOutput { return { type: "number", enum: [number.value] }; } diff --git a/packages/openapi3/test/string-template.test.ts b/packages/openapi3/test/string-template.test.ts new file mode 100644 index 0000000000..fe9c684f81 --- /dev/null +++ b/packages/openapi3/test/string-template.test.ts @@ -0,0 +1,66 @@ +import { expectDiagnostics } from "@typespec/compiler/testing"; +import { deepStrictEqual } from "assert"; +import { emitOpenApiWithDiagnostics, openApiFor } from "./test-host.js"; + +describe("openapi3: string templates", () => { + describe("handle interpolating literals", () => { + it("string", async () => { + const schemas = await openApiFor(` + model Test { + a: "Start \${"abc"} end", + } + `); + + deepStrictEqual(schemas.components?.schemas?.Test.properties.a, { + type: "string", + enum: ["Start abc end"], + }); + }); + + it("number", async () => { + const schemas = await openApiFor(` + model Test { + a: "Start \${123} end", + } + `); + + deepStrictEqual(schemas.components?.schemas?.Test.properties.a, { + type: "string", + enum: ["Start 123 end"], + }); + }); + + it("boolean", async () => { + const schemas = await openApiFor(` + model Test { + a: "Start \${true} end", + } + `); + + deepStrictEqual(schemas.components?.schemas?.Test.properties.a, { + type: "string", + enum: ["Start true end"], + }); + }); + }); + + it("emit diagnostics if interpolation value are not literals", async () => { + const [schemas, diagnostics] = await emitOpenApiWithDiagnostics(` + model Test { + a: "Start \${Bar} end", + } + model Bar {} + `); + + deepStrictEqual(schemas.components?.schemas?.Test.properties?.a, { + type: "string", + }); + + expectDiagnostics(diagnostics, { + code: "non-literal-string-template", + severity: "warning", + message: + "Value interpolated in this string template cannot be converted to a string. Only literal types can be automatically interpolated.", + }); + }); +}); diff --git a/packages/openapi3/test/test-host.ts b/packages/openapi3/test/test-host.ts index c238e3695a..b880319513 100644 --- a/packages/openapi3/test/test-host.ts +++ b/packages/openapi3/test/test-host.ts @@ -1,4 +1,4 @@ -import { interpolatePath } from "@typespec/compiler"; +import { Diagnostic, interpolatePath } from "@typespec/compiler"; import { createTestHost, createTestWrapper, @@ -9,8 +9,10 @@ import { HttpTestLibrary } from "@typespec/http/testing"; import { OpenAPITestLibrary } from "@typespec/openapi/testing"; import { RestTestLibrary } from "@typespec/rest/testing"; import { VersioningTestLibrary } from "@typespec/versioning/testing"; +import { ok } from "assert"; import { OpenAPI3EmitterOptions } from "../src/lib.js"; import { OpenAPI3TestLibrary } from "../src/testing/index.js"; +import { OpenAPI3Document } from "../src/types.js"; export async function createOpenAPITestHost() { return createTestHost({ @@ -47,6 +49,25 @@ export async function createOpenAPITestRunner({ }); } +export async function emitOpenApiWithDiagnostics( + code: string, + options: OpenAPI3EmitterOptions = {} +): Promise<[OpenAPI3Document, readonly Diagnostic[]]> { + const runner = await createOpenAPITestRunner(); + const outputFile = resolveVirtualPath("openapi.json"); + const diagnostics = await runner.diagnose(code, { + noEmit: false, + emit: ["@typespec/openapi3"], + options: { + "@typespec/openapi3": { ...options, "output-file": outputFile }, + }, + }); + const content = runner.fs.get(outputFile); + ok(content, "Expected to have found openapi output"); + const doc = JSON.parse(content); + return [doc, diagnostics]; +} + export async function diagnoseOpenApiFor(code: string, options: OpenAPI3EmitterOptions = {}) { const runner = await createOpenAPITestRunner(); const diagnostics = await runner.diagnose(code, { diff --git a/packages/spec/src/spec.emu.html b/packages/spec/src/spec.emu.html index 2e9e081ea0..46840457fc 100644 --- a/packages/spec/src/spec.emu.html +++ b/packages/spec/src/spec.emu.html @@ -154,7 +154,7 @@

Lexical Grammar

StringLiteral : `"` StringCharacters? `"` `"""` TripleQuotedStringCharacters? `"""` - StringTemplateHead + StringTemplate StringTemplate : TemplateHead Expression TemplateSpans From d0ae9657aab33e907cf5b78495da9df31cacfc8b Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Thu, 16 Nov 2023 14:33:35 -0800 Subject: [PATCH 23/24] openapi3 changelog --- .../feature-string-template_2023-11-16-22-33.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 common/changes/@typespec/openapi3/feature-string-template_2023-11-16-22-33.json diff --git a/common/changes/@typespec/openapi3/feature-string-template_2023-11-16-22-33.json b/common/changes/@typespec/openapi3/feature-string-template_2023-11-16-22-33.json new file mode 100644 index 0000000000..efcf1e2d84 --- /dev/null +++ b/common/changes/@typespec/openapi3/feature-string-template_2023-11-16-22-33.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@typespec/openapi3", + "comment": "Added support for string template literals", + "type": "none" + } + ], + "packageName": "@typespec/openapi3" +} \ No newline at end of file From 6c184055cdc1853254ec7f3fc09c1319d3dcc77c Mon Sep 17 00:00:00 2001 From: Timothee Guerin Date: Fri, 17 Nov 2023 11:58:17 -0800 Subject: [PATCH 24/24] ADd sample and fix issue with template param --- packages/compiler/src/core/checker.ts | 5 +- .../src/core/helpers/string-template-utils.ts | 34 +++++++++++++- .../samples/specs/string-template/main.tsp | 23 ++++++++++ .../@typespec/openapi3/openapi.yaml | 46 +++++++++++++++++++ 4 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 packages/samples/specs/string-template/main.tsp create mode 100644 packages/samples/test/output/string-template/@typespec/openapi3/openapi.yaml diff --git a/packages/compiler/src/core/checker.ts b/packages/compiler/src/core/checker.ts index 07146b1789..b2af6bcdc9 100644 --- a/packages/compiler/src/core/checker.ts +++ b/packages/compiler/src/core/checker.ts @@ -9,6 +9,7 @@ import { getTypeName, stringTemplateToString, } from "./helpers/index.js"; +import { isStringTemplateSerializable } from "./helpers/string-template-utils.js"; import { createDiagnostic } from "./messages.js"; import { getIdentifierContext, hasParseError, visitChildren } from "./parser.js"; import { Program, ProjectedProgram } from "./program.js"; @@ -3301,8 +3302,8 @@ export function createChecker(program: Program): Checker { return true; } if (type.kind === "StringTemplate") { - const [_, diagnostics] = stringTemplateToString(type); - return diagnostics.length === 0; + const [valid] = isStringTemplateSerializable(type); + return valid; } const valueTypes = new Set(["String", "Number", "Boolean", "EnumMember", "Tuple"]); return valueTypes.has(type.kind); diff --git a/packages/compiler/src/core/helpers/string-template-utils.ts b/packages/compiler/src/core/helpers/string-template-utils.ts index e6d1198d15..e466281a23 100644 --- a/packages/compiler/src/core/helpers/string-template-utils.ts +++ b/packages/compiler/src/core/helpers/string-template-utils.ts @@ -1,4 +1,4 @@ -import { createDiagnosticCollector } from "../index.js"; +import { createDiagnosticCollector } from "../diagnostics.js"; import { createDiagnostic } from "../messages.js"; import { Diagnostic, StringTemplate } from "../types.js"; import { getTypeName } from "./type-name-utils.js"; @@ -40,3 +40,35 @@ export function stringTemplateToString( .join(""); return diagnostics.wrap(result); } + +export function isStringTemplateSerializable( + stringTemplate: StringTemplate +): [boolean, readonly Diagnostic[]] { + const diagnostics = createDiagnosticCollector(); + for (const span of stringTemplate.spans) { + if (span.isInterpolated) { + switch (span.type.kind) { + case "String": + case "Number": + case "Boolean": + break; + case "StringTemplate": + diagnostics.pipe(isStringTemplateSerializable(span.type)); + break; + case "TemplateParameter": + if (span.type.constraint && span.type.constraint.kind === "Value") { + break; // Value types will be serializable in the template instance. + } + // eslint-disable-next-line no-fallthrough + default: + diagnostics.add( + createDiagnostic({ + code: "non-literal-string-template", + target: span.node, + }) + ); + } + } + } + return [diagnostics.diagnostics.length === 0, diagnostics.diagnostics]; +} diff --git a/packages/samples/specs/string-template/main.tsp b/packages/samples/specs/string-template/main.tsp new file mode 100644 index 0000000000..686031b287 --- /dev/null +++ b/packages/samples/specs/string-template/main.tsp @@ -0,0 +1,23 @@ +alias myconst = "foobar"; + +model Person { + simple: "Simple ${123} end"; + multiline: """ + Multi + ${123} + ${true} + line + """; + ref: "Ref this alias ${myconst} end"; + template: Template<"custom">; +} + +alias Template = "Foo ${T} bar"; + +/** Example of string template with template parameters */ +@doc("Animal named: ${T}") +model Animal { + kind: T; +} + +model Cat is Animal<"Cat">; diff --git a/packages/samples/test/output/string-template/@typespec/openapi3/openapi.yaml b/packages/samples/test/output/string-template/@typespec/openapi3/openapi.yaml new file mode 100644 index 0000000000..2291a05bc9 --- /dev/null +++ b/packages/samples/test/output/string-template/@typespec/openapi3/openapi.yaml @@ -0,0 +1,46 @@ +openapi: 3.0.0 +info: + title: (title) + version: 0000-00-00 +tags: [] +paths: {} +components: + schemas: + Cat: + type: object + required: + - kind + properties: + kind: + type: string + enum: + - Cat + description: 'Animal named: Cat' + Person: + type: object + required: + - simple + - multiline + - ref + - template + properties: + simple: + type: string + enum: + - Simple 123 end + multiline: + type: string + enum: + - |- + Multi + 123 + true + line + ref: + type: string + enum: + - Ref this alias foobar end + template: + type: string + enum: + - Foo custom bar