Added tests for new macro syntax

packages/parser/src/macros-v6/README.md

@@ -0,0 +1,4 @@
+## DoenetML v0.6 Macro Parser
+
+DoenetML v0.6 used a different macro format. This parser parses that older macro format.
+It is used by lsp-tools to convert macros to the newer format.

packages/parser/src/macros-v6/index.ts

@@ -0,0 +1,9 @@
+import { DastText, DastFunctionMacro, DastMacro } from "../types";
+import { MacroParser } from "./parser";
+
+/**
+ * Parse a string and turn it into a list of text/macro/function-macro nodes.
+ */
+export function parseMacrosV06(str: string): (DastText | DastMacro | DastFunctionMacro)[] {
+    return MacroParser.parse(str) as (DastText | DastMacro | DastFunctionMacro)[];
+}

packages/parser/src/macros-v6/macro-to-string.ts

@@ -0,0 +1,106 @@
+import { quote, toXml } from "../dast-to-xml/dast-util-to-xml";
+import {
+    Attr,
+    FullPath,
+    FunctionMacro,
+    Macro,
+    PropAccess,
+    ScopedPathPart,
+    Text,
+} from "./types";
+
+type Node = Macro | FunctionMacro | Text | PropAccess;
+
+/**
+ * Convert a "pure" macro to a string. I.e., a macro that was parsed directly from the peggy grammar.
+ * **Note**: This function is probably not what you want. You probably want `toXml`, since this function
+ * cannot print function macros that have XML nodes as children.
+ */
+export function macroToString(node: Node | Node[]): string {
+    if (Array.isArray(node)) {
+        return node.map((n) => macroToString(n)).join("");
+    }
+    switch (node.type) {
+        case "macro": {
+            const macro = unwrappedMacroToString(node);
+
+            let start = "$";
+            let end = "";
+            if (macroNeedsParens(node)) {
+                start += "(";
+                end += ")";
+            }
+            return start + macro + end;
+        }
+        case "function": {
+            const macro = unwrappedMacroToString(node.macro);
+
+            let start = "$$";
+            let end = "";
+            if (macroNeedsParens(node)) {
+                start += "(";
+                end += ")";
+            }
+            const args = node.input
+                ? `(${node.input.map(macroToString).join(", ")})`
+                : "";
+            return start + macro + end + args;
+        }
+        case "text":
+            return toXml(node);
+
+        default:
+            const _exhaustiveCheck: never = node;
+            console.warn("Unhandled node type", node);
+    }
+    return "$ERROR";
+}
+
+/**
+ * Convert a macro to a string, but do not wrap it in parens or add a `$` prefix.
+ */
+function unwrappedMacroToString(nodes: Macro): string {
+    const path = macroPathToString(nodes.path);
+    const attrs = (nodes.attributes || []).map(attrToString).join(" ");
+    let attrsStr = attrs.length > 0 ? `{${attrs}}` : "";
+    let propAccess = "";
+    if (nodes.accessedProp) {
+        propAccess = "." + unwrappedMacroToString(nodes.accessedProp);
+    }
+    return path + attrsStr + propAccess;
+}
+
+function macroPathToString(path: FullPath): string {
+    return path.map(macroPathPartToString).join("/");
+}
+
+function macroPathPartToString(pathPart: ScopedPathPart): string {
+    return (
+        pathPart.name +
+        pathPart.index.map((part) => `[${macroToString(part.value)}]`).join("")
+    );
+}
+
+function attrToString(attr: Attr): string {
+    const name = attr.name;
+    if (attr.children.length === 0) {
+        return name;
+    }
+    const value = attr.children.map(macroToString).join("");
+    return `${name}=${quote(value)}`;
+}
+
+function macroNeedsParens(macro: Macro | PropAccess | FunctionMacro): boolean {
+    if (macro.type === "function") {
+        return macroNeedsParens(macro.macro);
+    }
+    // Paths are separated by slashes. They always need wrapping.
+    if (macro.path.length > 1) {
+        return true;
+    }
+    // We also might need wrapping if the path contains a `-` character
+    return (
+        macro.path.some((part) => part.name.includes("-")) ||
+        (macro.accessedProp != null && macroNeedsParens(macro.accessedProp))
+    );
+}

packages/parser/src/macros-v6/macros.peggy

@@ -0,0 +1,201 @@
+{
+    function withPosition<const T>(node: T) {
+        const { start, end } = location();
+        return { ...node, position: { start, end } };
+    }
+}
+
+top = (Macro / FunctionMacro / Text)*
+
+// Identifiers. Scoped identifiers can only be used inside of `$(..)` notation.
+Ident = $[a-zA-Z0-9_]+
+
+ScopedIdent = $[a-zA-Z0-9_-]+
+
+// A PathPart cannot have slashes or `..` in it
+PathPart
+    = name:Ident index:PropIndex* {
+            return withPosition({ type: "pathPart", name, index });
+        }
+
+ScopedPathPart
+    = name:ScopedIdent index:PropIndex* {
+            return withPosition({ type: "pathPart", name, index });
+        }
+
+// A FullPath can have slashes and `..` in it. It may also start with a slash.
+EmptyPathPart
+    = "" { return withPosition({ type: "pathPart", name: "", index: [] }); }
+
+FullPath
+    = start:EmptyPathPart rest:("/" @ScopedPathPartOrDD)+ {
+            return [start, ...rest];
+        }
+    / start:ScopedPathPartOrDD rest:("/" @ScopedPathPartOrDD)* {
+            return [start, ...rest];
+        }
+
+ScopedPathPartOrDD
+    = ScopedPathPart
+    / ".." { return withPosition({ type: "pathPart", name: "..", index: [] }); }
+
+// Pops can be accessed with a `.`
+PropAccess = "." @PartialPathMacro
+
+ScopedPropAccess = "." @ScopedPartialPathMacro
+
+//
+// Macros
+//
+Macro
+    = "$" "(" macro:FullPathMacro ")" { return withPosition(macro) }
+    / "$" macro:PartialPathMacro { return withPosition(macro) }
+
+// A macro where the path cannot have slashes or `..` in it.
+PartialPathMacro
+    = path:PathPart attrs:PropAttrs? accessedProp:PropAccess? {
+            return withPosition({
+                type: "macro",
+                path: [path],
+                attributes: attrs || [],
+                accessedProp,
+            });
+        }
+
+ScopedPartialPathMacro
+    = path:ScopedPathPart attrs:PropAttrs? accessedProp:ScopedPropAccess? {
+            return withPosition({
+                type: "macro",
+                path: [path],
+                attributes: attrs || [],
+                accessedProp,
+            });
+        }
+
+// A macro where the path can have slashes and `..` in it.
+FullPathMacro
+    = path:FullPath attrs:PropAttrs? accessedProp:ScopedPropAccess? {
+            return withPosition({
+                type: "macro",
+                path,
+                attributes: attrs || [],
+                accessedProp,
+            });
+        }
+
+//
+// Functions
+//
+
+// Functions are very similar to macros except they cannot have attrs or accessedProps
+// but they do take comma-separated arguments.
+FunctionMacro
+    = "$$" "(" macro:FullPathMacro ")" input:FunctionInput? {
+            return withPosition({
+                type: "function",
+                macro,
+                input,
+            });
+        }
+    / "$$" macro:PartialPathMacro input:FunctionInput? {
+            return withPosition({
+                type: "function",
+                macro,
+                input,
+            });
+        }
+
+FunctionInput = "(" _? @FunctionArgumentList _? ")"
+
+FunctionArgumentList
+    = start:FunctionArgument rest:(_? "," _? @FunctionArgument)* {
+            return [start, ...rest];
+        }
+
+// A function argument cannot contain commas unless those commas are inside of balanced parens.
+// For example `$$f( (0,1) )` should be parsed as a function with a single argument.
+FunctionArgument = BalancedParenTextNoComma
+
+BalancedParenTextNoComma
+    = x:(
+        Macro
+        / FunctionMacro
+        / TextWithoutParenOrComma
+        / a:OpenParen rest:BalancedParenText b:CloseParen {
+                return [a, ...rest, b];
+            }
+    )* { return x.flat(); }
+    / x:EmptyString { return [x]; }
+
+BalancedParenText
+    = x:(
+        Macro
+        / FunctionMacro
+        / TextWithoutParen
+        / a:OpenParen rest:BalancedParenText b:CloseParen {
+                return [a, ...rest, b];
+            }
+    )* { return x.flat(); }
+    / x:EmptyString { return [x]; }
+
+PropAttrs = "{" _? @(@Attr _?)* "}"
+
+PropIndex
+    = "[" _? value:(@(FunctionMacro / Macro / TextWithoutClosingSquareBrace) _?)* "]" {
+            return withPosition({ type: "index", value });
+        }
+
+// Attribute stuff
+Attr
+    = name:AttrName _? "=" _? children:AttrValue {
+            return withPosition({ type: "attribute", name, children });
+        }
+    / name:AttrName {
+            return withPosition({
+                type: "attribute",
+                name,
+                children: [],
+            });
+        }
+
+AttrName = $[a-zA-Z0-9_:-]+
+
+AttrValue
+    = "\"" @(Macro / FunctionMacro / TextWithoutDoubleQuote)* "\""
+    / "'" @(Macro / FunctionMacro / TextWithoutQuote)* "'"
+
+// Different types of text with various restrictions
+Text = value:($[^$]+ / .) { return withPosition({ type: "text", value }); }
+
+EmptyString
+    = "" {
+            return withPosition({
+                type: "text",
+                value: "",
+            });
+        }
+
+OpenParen = "(" { return withPosition({ type: "text", value: "(" }); }
+
+CloseParen = ")" { return withPosition({ type: "text", value: ")" }); }
+
+TextWithoutParenOrComma
+    = value:($[^(),$]+ / [^(),]) {
+            return withPosition({ type: "text", value });
+        }
+
+TextWithoutParen
+    = value:($[^()$]+ / [^()]) { return withPosition({ type: "text", value }); }
+
+TextWithoutQuote
+    = value:$([^'$]+ / [^']) { return withPosition({ type: "text", value }); }
+
+TextWithoutClosingSquareBrace
+    = value:$([^\]$]+ / [^\]]) { return withPosition({ type: "text", value }); }
+
+TextWithoutDoubleQuote
+    = value:($[^"$]+ / [^"]) { return withPosition({ type: "text", value }); }
+
+_ = $[ \t\r\n]+
+
+EOF = !.

packages/parser/src/macros-v6/parser.ts

@@ -0,0 +1,19 @@
+// This file needs to be here because typescript does not know how to use the transpiler
+// to directly load Pegjs grammars.
+// @ts-nocheck
+import _MacroParser from "./macros.peggy";
+import { ParseFunction } from "./types";
+
+type PegParser = {
+    parse: ParseFunction;
+    SyntaxError: (
+        message: string,
+        expected: string,
+        found: unknown,
+        location: unknown,
+    ) => unknown;
+};
+
+const MacroParser = _MacroParser as PegParser;
+
+export { MacroParser };