Add operators support

atsky · Jun 5, 2014 · f4bdd39 · f4bdd39
1 parent ffd54a0
commit f4bdd39
Show file tree

Hide file tree

Showing 10 changed files with 704 additions and 605 deletions.
diff --git a/build.xml b/build.xml
@@ -3,24 +3,20 @@
         Tasks for Haskell plugin
     </description>
 
+    <taskdef classname="jflex.anttask.JFlexTask" name="jflex" classpath="lib/jflex-1.5.1.jar"/>
+
+
     <property name="result.file" value="plugin/src/org/jetbrains/haskell/parser/lexer/_HaskellLexer.java"/>
     <property name="flex.file" value="plugin/src/org/jetbrains/haskell/parser/lexer/Haskell.flex"/>
 
 
     <target name="run flex">
         <sequential>
             <delete file="${result.file}"/>
-            <java classname="JFlex.Main"
-                  jvmargs="-Xmx512M"
-                  failonerror="true">
-                <arg value="--charat"/>
-                <arg value="-skel"/>
-                <arg value="tools/idea-flex.skeleton"/>
-                <arg value="${flex.file}"/>
-                <classpath>
-                    <pathelement location="lib/JFlex.jar"/>
-                </classpath>
-            </java>
+            <jflex  file="${flex.file}"
+                    skeleton="tools/idea-flex.skeleton"
+
+                    />
         </sequential>
     </target>
 

diff --git a/lib/JFlex.jar b/lib/JFlex.jar
diff --git a/lib/jflex-1.5.1.jar b/lib/jflex-1.5.1.jar
diff --git a/plugin/src/org/jetbrains/haskell/parser/grammar/ExpressionsGrammar.kt b/plugin/src/org/jetbrains/haskell/parser/grammar/ExpressionsGrammar.kt
@@ -6,7 +6,7 @@ import org.jetbrains.haskell.parser.token.COLON
 import org.jetbrains.haskell.parser.token.STRING
 import org.jetbrains.haskell.parser.token.NUMBER
 import org.jetbrains.haskell.parser.token.DOT
-import org.jetbrains.haskell.parser.token.OPERATOR
+import org.jetbrains.haskell.parser.token.OPERATOR_ID
 import org.jetbrains.haskell.parser.token.DOLLAR
 import org.jetbrains.haskell.parser.rules.rule
 import org.jetbrains.haskell.parser.token.TYPE_OR_CONS
@@ -50,7 +50,7 @@ val anAtomExpression = lazy {
     NUMBER or
     REFERENCE_EXPRESSION or
     DOT or
-    OPERATOR or
+    OPERATOR_ID or
     DOLLAR or
     FIELD_UPDATE or
     CASE_EXPRESSION or

diff --git a/plugin/src/org/jetbrains/haskell/parser/grammar/HaskellGrammar.kt b/plugin/src/org/jetbrains/haskell/parser/grammar/HaskellGrammar.kt
@@ -107,7 +107,7 @@ private val aModuleExports = rule(MODULE_EXPORTS) {
 
 val anExport = lazy {
     val symbolExport = rule(SYMBOL_EXPORT) {
-        ID or TYPE_OR_CONS or inParentheses(OPERATOR)
+        ID or TYPE_OR_CONS or inParentheses(OPERATOR_ID)
     }
 
     val qcnameExt = maybe(TYPE_KW) + symbolExport
@@ -158,7 +158,7 @@ val aDataDeclaration = rule(DATA_DECLARATION) {
 }
 
 val SOME_ID = RuleBasedElementType("Some id", ::SomeId) {
-    ID or TYPE_OR_CONS or OPERATOR
+    ID or TYPE_OR_CONS or OPERATOR_ID
 }
 
 val ANY : Rule = RuleBasedElementType("Any", ::UnparsedToken) {

diff --git a/plugin/src/org/jetbrains/haskell/parser/lexer/Haskell.flex b/plugin/src/org/jetbrains/haskell/parser/lexer/Haskell.flex
@@ -36,6 +36,7 @@ digit     = {ascdigit}|{unidigit}
 
 special   = [\(\)\,\;\[\]\`\{\}]
 ascsymbol = [\!\#\$\%\&\*\+\.\/\<\=\>\?\@\\\^\|\-\~]
+unisymbol = [[\p{P}\p{S}]&&[^(),;\[\]`{}_\"\']]
 symbol    = {ascsymbol}|{unisymbol}
 
 large     = [:uppercase:]
@@ -133,6 +134,8 @@ UCHARACTER = (\'\\x[0-9]*\')
 "=>"                  { return TokenPackage.getDOUBLE_ARROW(); }
 "!"                   { return TokenPackage.getEXCLAMATION(); }
 "_"                   { return TokenPackage.getUNDERSCORE(); }
+":"{symbol}+          { return TokenPackage.getOPERATOR_CONS(); }
+{symbol}+             { return TokenPackage.getOPERATOR_ID(); }
 
 // - Keywords
 

diff --git a/plugin/src/org/jetbrains/haskell/parser/lexer/_HaskellLexer.java b/plugin/src/org/jetbrains/haskell/parser/lexer/_HaskellLexer.java
diff --git a/plugin/src/org/jetbrains/haskell/parser/token/HaskelLexerTokens.kt b/plugin/src/org/jetbrains/haskell/parser/token/HaskelLexerTokens.kt
@@ -109,7 +109,8 @@ public val BLOCK_COMMENT: HaskellToken        = HaskellToken("COMMENT")
 public val END_OF_LINE_COMMENT : HaskellToken = HaskellToken("--")
 public val ID : HaskellToken                  = HaskellToken("id")
 public val NUMBER : HaskellToken              = HaskellToken("number")
-public val OPERATOR : HaskellToken            = HaskellToken("opertor")
+public val OPERATOR_ID: HaskellToken          = HaskellToken("opertor")
+public val OPERATOR_CONS: HaskellToken        = HaskellToken("opertor cons")
 public val PRAGMA : HaskellToken              = HaskellToken("PRAGMA")
 public val STRING : HaskellToken              = HaskellToken("string")
 public val TYPE_OR_CONS: HaskellToken         = HaskellToken("type_cons")

diff --git a/plugin/test/org/jetbrains/haskell/lexer/HaskellLexerTest.java b/plugin/test/org/jetbrains/haskell/lexer/HaskellLexerTest.java
@@ -61,6 +61,12 @@ public void testQuotation() throws Exception {
                 "Haskell Token:id ('name')");
     }
 
+    @Test
+    public void testOperators() throws Exception {
+        doTest("\u222F",
+                "Haskell Token:opertor ('\u222F')");
+    }
+
     @Test
     public void testIndentsComments() throws Exception {
         doTest("module Main where\n" +

diff --git a/tools/idea-flex.skeleton b/tools/idea-flex.skeleton
@@ -28,7 +28,7 @@
 
   /** this buffer contains the current text to be matched and is
       the source of the yytext() string */
-  private CharSequence zzBuffer = "";
+  private char[] zzBuffer = new char[0];
 
   /** this buffer may contains the current text array to be matched when it is cheap to acquire it */
   private char[] zzBufferArray;
@@ -70,7 +70,7 @@
   }
 
   public void reset(CharSequence buffer, int start, int end,int initialState){
-    zzBuffer = buffer;
+    zzBuffer = buffer.toString().toCharArray();
     zzBufferArray = com.intellij.util.text.CharArrayUtil.fromSequenceWithoutCopying(buffer);
     zzCurrentPos = zzMarkedPos = zzStartRead = start;
     zzPushbackPos = 0;
@@ -114,7 +114,7 @@
    * Returns the text matched by the current regular expression.
    */
   public final CharSequence yytext() {
-    return zzBuffer.subSequence(zzStartRead, zzMarkedPos);
+    return new String(zzBuffer, zzStartRead, zzMarkedPos);
   }
 
 
@@ -130,7 +130,7 @@
    * @return the character at position pos
    */
   public final char yycharat(int pos) {
-    return zzBufferArray != null ? zzBufferArray[zzStartRead+pos]:zzBuffer.charAt(zzStartRead+pos);
+    return zzBufferArray != null ? zzBufferArray[zzStartRead+pos]:zzBuffer[zzStartRead+pos];
   }
 
 
@@ -201,7 +201,7 @@
     int zzCurrentPosL;
     int zzMarkedPosL;
     int zzEndReadL = zzEndRead;
-    CharSequence zzBufferL = zzBuffer;
+    char[] zzBufferL = zzBuffer;
     char[] zzBufferArrayL = zzBufferArray;
     char [] zzCMapL = ZZ_CMAP;