From 5a7255dd73c0e55c1e997b6d01c88278a8343e63 Mon Sep 17 00:00:00 2001 From: Clayton Carter Date: Wed, 27 Apr 2022 20:41:58 -0400 Subject: [PATCH] Add injections and wrapping grammars for tree-sitter grammar --- grammars/tree-sitter-html.cson | 36 +++++++++++ grammars/tree-sitter-phpdoc.cson | 18 ++++++ lib/main.js | 28 +++++++++ package-lock.json | 16 +++++ package.json | 3 + spec/tree-sitter-helpers.js | 101 +++++++++++++++++++++++++++++++ spec/tree-sitter-html-spec.js | 74 ++++++++++++++++++++++ 7 files changed, 276 insertions(+) create mode 100644 grammars/tree-sitter-html.cson create mode 100644 grammars/tree-sitter-phpdoc.cson create mode 100644 lib/main.js create mode 100644 spec/tree-sitter-helpers.js create mode 100644 spec/tree-sitter-html-spec.js diff --git a/grammars/tree-sitter-html.cson b/grammars/tree-sitter-html.cson new file mode 100644 index 0000000..e08248f --- /dev/null +++ b/grammars/tree-sitter-html.cson @@ -0,0 +1,36 @@ +# This grammar is responsible for injecting the actual PHP or HTML grammars +# For the actual PHP scopes, see tree-sitter-php.cson +name: 'PHP' +scopeName: 'text.html.php' +type: 'tree-sitter' +parser: 'tree-sitter-embedded-php' + +fileTypes: [ + 'aw' + 'ctp' + 'inc' + 'install' + 'module' + 'php' + 'php_cs' + 'php3' + 'php4' + 'php5' + 'phpt' + 'phtml' + 'profile' +] + +firstLineRegex: [ + '^\\s*<\\?([pP][hH][pP]|=|\\s|$)' +] + +scopes: + 'template > content:nth-child(0)': [ + {match: /^\#!.*(?:\s|\/)php\d?(?:$|\s)/, scopes: 'comment.line.shebang.php'} + ], + + 'php': [ + {match: '\\n', scopes: 'meta.embedded.block.php'} + 'meta.embedded.line.php' + ] diff --git a/grammars/tree-sitter-phpdoc.cson b/grammars/tree-sitter-phpdoc.cson new file mode 100644 index 0000000..a79f589 --- /dev/null +++ b/grammars/tree-sitter-phpdoc.cson @@ -0,0 +1,18 @@ +name: 'PHPDoc' +scopeName: 'comment.block.documentation.phpdoc.php' +type: 'tree-sitter' +parser: 'tree-sitter-phpdoc' + +injectionRegex: 'phpdoc|PHPDoc' + +scopes: + tag_name: 'keyword.other.phpdoc.php' + type_list: 'meta.other.type.phpdoc.php' + 'type_list > "|"': 'punctuation.separator.delimiter.php' + 'array_type > "[]"': 'keyword.other.array.phpdoc.php' + primitive_type: 'keyword.other.type.php' + 'named_type > name': 'support.class.php' + '* > namespace_name_as_prefix': 'support.other.namespace.php' + # FIXME not working + # '* > namespace_name_as_prefix > "\\"': 'punctuation.separator.inheritance.php' + '* > qualified_name > name': 'support.class.php' diff --git a/lib/main.js b/lib/main.js new file mode 100644 index 0000000..cceb6a7 --- /dev/null +++ b/lib/main.js @@ -0,0 +1,28 @@ +exports.activate = function() { + if (!atom.grammars.addInjectionPoint) return + + // inject source.php into text.html.php + atom.grammars.addInjectionPoint('text.html.php', { + type: 'php', + language () { return 'php' }, + content (php) { return php } + }) + + // inject html into text.html.php + atom.grammars.addInjectionPoint('text.html.php', { + type: 'template', + language () { return 'html' }, + content (node) { return node.descendantsOfType('content') } + }) + + // inject phpDoc comments into PHP comments + atom.grammars.addInjectionPoint('source.php', { + type: 'comment', + language (comment) { + if (comment.text.startsWith('/**') && !comment.text.startsWith('/***')) { + return 'phpdoc' + } + }, + content (comment) { return comment } + }) +} diff --git a/package-lock.json b/package-lock.json index 58a9e11..e74478c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -151,6 +151,14 @@ "integrity": "sha1-HhX7ysl9Pumb8tc7TGVrCCu6+5E=", "dev": true }, + "tree-sitter-embedded-php": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/tree-sitter-embedded-php/-/tree-sitter-embedded-php-0.0.4.tgz", + "integrity": "sha512-JdINmNoncwZgA7QytmPy0TLrp6tJ7p9nyHZvUUNs7ndV4QtLdIYnRJRIAFTosH4Io1GO6Ok6HN9xsX/5sGCdRA==", + "requires": { + "nan": "^2.14.0" + } + }, "tree-sitter-php-abc": { "version": "0.17.0", "resolved": "https://registry.npmjs.org/tree-sitter-php-abc/-/tree-sitter-php-abc-0.17.0.tgz", @@ -159,6 +167,14 @@ "nan": "^2.14.0" } }, + "tree-sitter-phpdoc": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/tree-sitter-phpdoc/-/tree-sitter-phpdoc-0.0.4.tgz", + "integrity": "sha512-vHJ7dp/l8i6XCiYmQAWRj274ph/wc+JxQV9g3CpJDL3/4hxYN+yjWhRY4rmLQzA2IjWFWKbNwJjJ3+C5A9JodA==", + "requires": { + "nan": "^2.14.0" + } + }, "wordwrap": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", diff --git a/package.json b/package.json index f8e6bc5..7f1d3e6 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "atom": "*", "node": "*" }, + "main": "lib/main", "homepage": "http://atom.github.io/language-php", "repository": { "type": "git", @@ -16,7 +17,9 @@ "url": "https://github.com/atom/language-php/issues" }, "dependencies": { + "tree-sitter-embedded-php": "0.0.4", "tree-sitter-php-abc": "^0.17.0", + "tree-sitter-phpdoc": "0.0.4" }, "devDependencies": { "coffeelint": "^1.10.1", diff --git a/spec/tree-sitter-helpers.js b/spec/tree-sitter-helpers.js new file mode 100644 index 0000000..c6d3400 --- /dev/null +++ b/spec/tree-sitter-helpers.js @@ -0,0 +1,101 @@ +const dedent = require("dedent"); + +module.exports = { + // https://github.com/atom/atom/blob/b3d3a52d9e4eb41f33df7b91ad1f8a2657a04487/spec/tree-sitter-language-mode-spec.js#L47-L55 + expectTokensToEqual(editor, expectedTokenLines, startingRow = 1) { + const lastRow = editor.getLastScreenRow(); + + for (let row = startingRow; row <= lastRow - startingRow; row++) { + const tokenLine = editor + .tokensForScreenRow(row) + .map(({ text, scopes }) => ({ + text, + scopes: scopes.map((scope) => + scope + .split(" ") + .map((className) => className.replace("syntax--", "")) + .join(".") + ), + })); + + const expectedTokenLine = expectedTokenLines[row - startingRow]; + + expect(tokenLine.length).toEqual(expectedTokenLine.length); + for (let i = 0; i < tokenLine.length; i++) { + expect(tokenLine[i].text).toEqual( + expectedTokenLine[i].text, + `Token ${i}, row: ${row}` + ); + expect(tokenLine[i].scopes).toEqual( + expectedTokenLine[i].scopes, + `Token ${i}, row: ${row}, token: '${tokenLine[i].text}'` + ); + } + } + }, + + toHaveScopesAtPosition(posn, token, expected, includeEmbeddedScopes = false) { + if (expected === undefined) { + expected = token; + } + if (token === undefined) { + expected = []; + } + + // token is not used at this time; it's just a way to keep note where we are + // in the line + + let filterEmbeddedScopes = (scope) => + includeEmbeddedScopes || + (scope !== "text.html.php" && + scope !== "meta.embedded.block.php" && + scope !== "meta.embedded.line.php"); + + let actual = this.actual + .scopeDescriptorForBufferPosition(posn) + .scopes.filter(filterEmbeddedScopes); + + let notExpected = actual.filter((scope) => !expected.includes(scope)); + let notReceived = expected.filter((scope) => !actual.includes(scope)); + + let pass = notExpected.length === 0 && notReceived.length === 0; + + if (pass) { + this.message = () => "Scopes matched"; + } else { + let line = this.actual.getBuffer().lineForRow(posn[0]); + let caret = " ".repeat(posn[1]) + "^"; + + this.message = () => + `Failure: + Scopes did not match at position [${posn.join(", ")}]: +${line} +${caret} + These scopes were expected but not received: + ${notReceived.join(", ")} + These scopes were received but not expected: + ${notExpected.join(", ")} + `; + } + + return pass; + }, + + setPhpText(content) { + this.setText(` { + const subscription = editor + .getBuffer() + .getLanguageMode() + .onDidChangeHighlighting(() => { + subscription.dispose(); + resolve(); + }); + }); + }, +}; diff --git a/spec/tree-sitter-html-spec.js b/spec/tree-sitter-html-spec.js new file mode 100644 index 0000000..d39b391 --- /dev/null +++ b/spec/tree-sitter-html-spec.js @@ -0,0 +1,74 @@ +const dedent = require("dedent"); +const {expectTokensToEqual, toHaveScopesAtPosition, nextHighlightingUpdate} = require('./tree-sitter-helpers') + +describe("Tree-sitter PHP grammar", () => { + var editor; + + beforeEach(async () => { + atom.config.set("core.useTreeSitterParsers", true); + await atom.packages.activatePackage("language-php"); + await atom.packages.activatePackage("language-html"); + editor = await atom.workspace.open("foo.php"); + }); + + beforeEach(function () { + this.addMatchers({ toHaveScopesAtPosition }); + }); + + describe("loading the grammar", () => { + it('loads the wrapper HTML grammar', () => { + embeddingGrammar = atom.grammars.grammarForScopeName("text.html.php"); + expect(embeddingGrammar).toBeTruthy(); + expect(embeddingGrammar.scopeName).toBe("text.html.php"); + expect(embeddingGrammar.constructor.name).toBe("TreeSitterGrammar"); + // FIXME how to test that all selectors were loaded correctly? Invalid + // selectors may generate errors and it would be great to catch those here. + + // injections + expect(embeddingGrammar.injectionPointsByType.template).toBeTruthy(); + expect(embeddingGrammar.injectionPointsByType.php).toBeTruthy(); + }) + }); + + describe("shebang", () => { + it("recognises shebang on the first line of document", () => { + editor.setText(dedent` + #!/usr/bin/env php + + `); + + // expect(editor).toHaveScopesAtPosition([0, 0], "#!", ["text.html.php", "comment.line.shebang.php", "punctuation.definition.comment.php"], true); + expect(editor).toHaveScopesAtPosition([0, 1], "#!", ["text.html.php", "comment.line.shebang.php", + // FIXME following scopes differ from TM + 'source.html' + ], true); + expect(editor).toHaveScopesAtPosition([0, 2], "/usr/bin/env php", ["text.html.php", "comment.line.shebang.php", + // FIXME following scopes differ from TM + 'source.html' + ], true); + expect(editor).toHaveScopesAtPosition([1, 0], " { + editor.setText(dedent` + + #!/usr/bin/env php + + `); + + expect(editor).toHaveScopesAtPosition([1, 0], "#!", ["text.html.php"], true); + expect(editor).toHaveScopesAtPosition([1, 1], "#!", ["text.html.php", + // FIXME following scopes differ from TM + 'meta.embedded.line.php', + 'source.php', + 'punctuation.section.embedded.begin.php' + ], true); + }); + }); +});