feat: add syntax highligthing to lezer grammar (#198)

* feat: add syntax highligthing to lezer grammar * add If EndIf as keywords * fix: scenario highlighting * fix: now comment is seen as a statement * fix: and color * add Foreach statement * fix: add newline to the end of the statements * fix: errors wrong colors * fix:update test cases * fix:add missing licenses
dyne · Sep 6, 2024 · ab68052 · ab68052
1 parent b812202
commit ab68052
Show file tree

Hide file tree

Showing 6 changed files with 346 additions and 244 deletions.
diff --git a/.gitignore b/.gitignore
@@ -77,6 +77,7 @@ node_modules
 !/grammar/rollup.config.js
 !/grammar/tsconfig.json
 !/grammar/.releaserc
-
+!/grammar/src/syntax.grammar.terms.d.ts
+!/grammar/src/tokens.js
 
 
diff --git a/grammar/src/index.ts b/grammar/src/index.ts
@@ -3,26 +3,35 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 import { parser } from "./syntax.grammar"
-import { LRLanguage, LanguageSupport, indentNodeProp, foldNodeProp, foldInside, delimitedIndent } from "@codemirror/language"
+import { LRLanguage, LanguageSupport, HighlightStyle, syntaxHighlighting } from "@codemirror/language"
 import { styleTags, tags as t } from "@lezer/highlight"
 import { completeFromList } from "@codemirror/autocomplete"
 
+const syntax_colors = syntaxHighlighting(
+	HighlightStyle.define(
+	  [
+		{ tag: t.heading, color: "purple" },
+		{ tag: t.heading1, color: "gray" },
+		{tag: t.variableName, color: "red"},
+		{ tag: t.keyword, color: "green" },
+		{tag: t.string, color: "blue"},
+		{tag: t.lineComment, color: "gray"},
+		{tag: t.heading2, color: "black"}
+	  ],
+	  { all: { color: "black" } }
+	)
+  );
 export const SlangroomLanguage = LRLanguage.define({
 	parser: parser.configure({
 		props: [
-			indentNodeProp.add({
-				Application: delimitedIndent({ closing: ")", align: false })
-			}),
-			foldNodeProp.add({
-				Application: foldInside
-			}),
 			styleTags({
-				Identifier: t.variableName,
-				WhenStatement: t.variableName,
-				ThenStatement: t.bool,
-				Rule: t.string,
-				StringLitteral: t.string,
-				LineComment: t.lineComment
+				"Given Then When and in inside If EndIf Foreach EndForeach" : t.variableName,
+				"have send open connect print output" : t.keyword,
+				"Rule VersionRule! UnknownIgnoreRule! GenericRule!": t.heading,
+				" Scenario ScenarioType/... ScenarioComment!": t.heading1,
+				"DbAction! EthereumAction! FsAction! GitAction! HelpersAction! HttpAction! JsonSchemaAction! OAuthAction! PocketbaseAction! QrCodeAction! RedisAction! ShellAction! TimestampAction! WalletAction! ZencodeAction!": t.heading2,
+				StringLiteral: t.string,
+				Comment: t.lineComment,
 			})
 		]
 	}),
@@ -31,7 +40,7 @@ export const SlangroomLanguage = LRLanguage.define({
 	}
 })
 
-export const ac = SlangroomLanguage.data.of({
+const ac = SlangroomLanguage.data.of({
 	autocomplete: completeFromList([
 		{ label: "given", type: "keyword" },
 		{ label: "then", type: "keyword" },
@@ -40,5 +49,5 @@ export const ac = SlangroomLanguage.data.of({
 })
 
 export function Slangroom() {
-	return new LanguageSupport(SlangroomLanguage, [ac])
+	return new LanguageSupport(SlangroomLanguage, [syntax_colors, ac])
 }
diff --git a/grammar/src/syntax.grammar b/grammar/src/syntax.grammar
@@ -3,21 +3,30 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 @top Statement {
-    ( RuleStatement? ScenarioStatement* RuleStatement?  SlangroomStatement)
+    ( RuleStatement | ScenarioStatement | RuleStatement |  SlangroomStatement | Comment )*
+}
+
+Comment {
+  comment (newline+ |eof)
 }
 
 SlangroomStatement {
-    (GivenStatement | ThenStatement | ThenPrint | GivenHaveStatement | GivenName | WhenStatement | IfEndifStatement)*
+    (GivenStatement | ThenStatement | ThenPrint | GivenHaveStatement | GivenName | WhenStatement | IfEndifStatement |ForEachStatement)
 }
 
 RuleStatement {
-    kw<"Rule"> (VersionRule | UnknownIgnoreRule | GenericRule)
+    kw<"Rule"> (VersionRule | UnknownIgnoreRule | GenericRule) ( newline+ | eof )
 }
 
 ScenarioStatement {
-    kw<"Scenario"> (StringLiteral| Identifier | Keywords) ":"? (Identifier | Keywords)*
+    kw<"Scenario"> ScenarioType ScenarioComment? ( newline+ | eof )
+}
+ScenarioType {
+  (StringLiteral| Identifier | Keywords)
+}
+ScenarioComment {
+	":" (Identifier | Keywords | StringLiteral)*
 }
-
 VersionRule {
     kw<"version"> VersionNumber
 }
@@ -35,50 +44,57 @@ VersionNumber {
 }
 
 @skip {
-    space | newline | comment
+    space
 }
 
 GivenStatement {
     kw<"Given"> kw<"I">
     (DbStatement | EthereumStatement | FsStatement | GitStatement | HelpersStatement | HttpStatement | JsonSchemaStatement | OAuthStatement | PocketbaseStatement | QrCodeStatement | RedisStatement | ShellStatement | TimestampStatement | WalletStatement | ZencodeStatement)
+	( newline+ | eof )
 }
 
 ThenStatement {
     kw<"Then"> kw<"I">
     (DbStatement | EthereumStatement | FsStatement | GitStatement | HelpersStatement | HttpStatement | JsonSchemaStatement | OAuthStatement | PocketbaseStatement | QrCodeStatement | RedisStatement | ShellStatement | TimestampStatement | WalletStatement | ZencodeStatement)
+	( newline+ | eof )
 }
 
 ThenPrint {
-    kw<"Then"> kw<"I">? kw<"print"> (StringLiteral | Identifier | Keywords)+ (kw<"I"> kw<"print"> (StringLiteral | Identifier | Keywords)+)*
+    kw<"Then"> kw<"I">? kw<"print"> (StringLiteral | Identifier | Keywords)+ (newline kw<"and"> +kw<"I"> kw<"print"> (StringLiteral | Identifier | Keywords)+)* ( newline+ | eof )
 }
 
 GivenHaveStatement {
     kw<"Given"> kw<"that">? kw<"I"> kw<"have"> (kw<"a">? | kw<"my">? | kw<"the">?) kw<"valid">? StringLiteral kw<"named">? StringLiteral? (kw<"inside"> | kw<"in">)?  kw<"named">? StringLiteral?
-    (kw <"and"> kw<"I"> kw<"have"> (kw<"a">? | kw<"my">?) kw<"valid">? StringLiteral kw<"named">? StringLiteral? (kw<"inside"> | kw<"in">)? kw<"named">? StringLiteral?)*
+    (newline* kw <"and"> kw<"I"> kw<"have"> (kw<"a">? | kw<"my">?) kw<"valid">? StringLiteral kw<"named">? StringLiteral? (kw<"inside"> | kw<"in">)? kw<"named">? StringLiteral?)* ( newline+ | eof )
 }
 HaveStatement{
-	kw<"and"> kw<"I"> kw<"have"> (kw<"a">? | kw<"my">?) kw<"valid">? StringLiteral kw<"named">? StringLiteral? (kw<"inside"> | kw<"in">)? kw<"named">? StringLiteral?
+	 newline+ kw<"and"> kw<"I"> kw<"have"> (kw<"a">? | kw<"my">?) kw<"valid">? StringLiteral kw<"named">? StringLiteral? (kw<"inside"> | kw<"in">)? kw<"named">? StringLiteral?
 }
 
 GivenName {
-    (kw<"Given"> kw<"I"> kw<"am">
+    ((kw<"Given"> kw<"I"> kw<"am">
     (kw<"known"> kw<"as">)? StringLiteral) |
 	(kw<"Given">  kw<"I">? kw<"my"> kw<"name"> kw<"is">
-	 (Identifier | Keywords | StringLiteral)+) (HaveStatement)*
+	 (Identifier | Keywords | StringLiteral)+)) HaveStatement* ( newline+ | eof )
 
 }
 
 WhenStatement {
-  kw<"When"> kw<"I"> (Identifier | Keywords | StringLiteral)+ (kw<"I"> (Identifier | Keywords | StringLiteral)+)*
+  kw<"When"> kw<"I"> (Identifier | Keywords | StringLiteral)+ (newline kw<"and"> kw<"I"> (Identifier | Keywords | StringLiteral)+)* (newline+ | eof)
 }
 
 IfEndifStatement{
-    Condition+ (WhenStatement |  ThenStatement | ThenPrint)* kw<"EndIf">
+    Condition+ ( GivenStatement | WhenStatement |  ThenStatement | ThenPrint)* kw<"EndIf"> ( newline+ | eof )
 }
 
 Condition {
-    kw<"If"> kw<"I"> kw<"verify"> (Identifier | Keywords | StringLiteral)+
- }
+    kw<"If"> kw<"I"> kw<"verify"> (Identifier | Keywords | StringLiteral)+ ( newline+ | eof )
+}
+
+ForEachStatement {
+	kw<"Foreach"> (StringLiteral | Identifier | Keywords) (kw<"in"> | kw<"inside">) (StringLiteral | Identifier | Keywords)+  newline+ (GivenStatement | WhenStatement |  ThenStatement | ThenPrint)*
+	kw<"EndForeach"> ( newline+ | eof )
+}
 
 // ===== Plugin-Specific Statements =====
 
@@ -420,15 +436,18 @@ Keywords {
     kw<"redis"> | kw<"delete"> | kw<"shell"> | kw<"fetch"> |
     kw<"local"> | kw<"timestamp"> | kw<"milliseconds"> | kw<"seconds"> |
     kw<"vc"> | kw<"sd"> | kw<"jwt"> | kw<"p-256"> | kw<"public"> |
-    kw<"pretty"> | kw<"print"> | kw<"zencode"> | kw<"output"> | kw<"is">
+    kw<"pretty"> | kw<"print"> | kw<"zencode"> | kw<"output"> | kw<"is"> |
+	kw<"Given"> | kw<"Then"> | kw<"When"> | kw<"Rule"> | kw<"Scenario"> | kw<"If"> | kw<"EndIf"> | kw<"Foreach"> | kw<"EndForeach"> | kw<"I">
 }
 
 @tokens {
     space { " " | "\t" }
     newline { "\n" | "\r\n" }
-    comment { "#" (![\n])* }
+    comment { "#" (![\n])*}
     StringLiteral { "'" (![\\\n'] | "\\" _)* "'" }
     symbols { $[a-zA-Z_] | "_" }
     Identifier { symbols+ Number* }
     Number { $[0-9]+ }
 }
+
+@external tokens token from "./tokens.js" { eof }
diff --git a/grammar/src/syntax.grammar.terms.d.ts b/grammar/src/syntax.grammar.terms.d.ts
@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2024 Dyne.org foundation
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+export const eof: number
diff --git a/grammar/src/tokens.js b/grammar/src/tokens.js
@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: 2024 Dyne.org foundation
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import {ExternalTokenizer} from "@lezer/lr"
+import { eof } from "./syntax.grammar.terms"
+
+export const token = new ExternalTokenizer(
+	(input) => {
+	  if (input.next < 0) {
+		input.acceptToken(eof);
+	  }
+	},
+	{ contextual: true, fallback: true }
+  );