Merge branch 'master' into Issue_1666

.gitignore

@@ -31,6 +31,9 @@ __pycache__/
 
 # User-specific files (MonoDevelop/Xamarin Studio)
 *.userprefs
+*.user
+.vs/
+project.lock.json
 
 # Build results
 [Dd]ebug/

.travis.yml

@@ -45,12 +45,22 @@ matrix:
     - os: linux
       jdk: oraclejdk7
       env: TARGET=csharp
+    - os: linux
+      jdk: oraclejdk7
+      dist: trusty
+      env: TARGET=dotnet
     - os: linux
       jdk: oraclejdk7
       env: TARGET=python2
     - os: linux
       jdk: oraclejdk7
       env: TARGET=python3
+      addons:
+        apt:
+          sources:
+            - deadsnakes # source required so it finds the package definition below
+          packages:
+            - python3.5
     - os: linux
       jdk: oraclejdk7
       env: TARGET=javascript

.travis/before-install-linux-cpp.sh

@@ -3,6 +3,4 @@
 set -euo pipefail
 
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
 sudo apt-get update -qq

.travis/before-install-linux-csharp.sh

@@ -3,8 +3,6 @@
 set -euo pipefail
 
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
 sudo apt-get update -qq
 echo "deb http://download.mono-project.com/repo/debian wheezy/snapshots/3.12.1 main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list
 sudo apt-get install -qq mono-complete

.travis/before-install-linux-dotnet.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# install dotnet
+sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ trusty main" > /etc/apt/sources.list.d/dotnetdev.list'
+sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 417A0893
+sudo apt-get update
+
+sudo apt-get install dotnet-dev-1.0.0-preview2.1-003177
+
+# install mvn
+wget http://apache.mirrors.lucidnetworks.net/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz && \
+    wget https://www.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz.md5 && \
+    echo "$(cat apache-maven-3.3.9-bin.tar.gz.md5)  apache-maven-3.3.9-bin.tar.gz" > apache-maven-3.3.9-bin.tar.gz.md5 && \
+    md5sum -c *.md5
+
+sudo rm -rf /usr/local/maven/ && sudo mkdir -p /usr/local/maven && \
+    sudo tar xzvf apache-maven-3.3.9-bin.tar.gz -C /usr/local/maven --strip-components=1
+
+mvn -v
+

.travis/before-install-linux-go.sh

@@ -3,8 +3,6 @@
 set -euo pipefail
 
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
 sudo apt-get update -qq
 eval "$(sudo gimme 1.7.3)"
 ( go version ; go env ) || true

.travis/before-install-linux-java.sh

@@ -3,6 +3,4 @@
 set -euo pipefail
 
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
 sudo apt-get update -qq

.travis/before-install-linux-javascript.sh

@@ -3,7 +3,7 @@
 set -euo pipefail
 
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
 sudo apt-get update -qq
+curl -sL https://deb.nodesource.com/setup_0.12 | sudo -E bash -
 sudo apt-get install -qq nodejs
+node --version

.travis/before-install-linux-python2.sh

@@ -3,7 +3,5 @@
 set -euo pipefail
 
 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
 sudo apt-get update -qq
 python --version

.travis/before-install-linux-python3.sh

@@ -2,9 +2,4 @@
 
 set -euo pipefail
 
-sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-sudo add-apt-repository ppa:fkrull/deadsnakes -y
-sudo add-apt-repository ppa:rwky/nodejs -y
-sudo apt-get update -qq
-sudo apt-get install -qq python3.5
 python3 --version

.travis/run-tests-dotnet.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+mvn -q -Dparallel=methods -DthreadCount=4 -Dtest=csharp.* -DargLine="-Dantlr-csharp-netstandard=true" test
+

CONTRIBUTING.md

@@ -0,0 +1,10 @@
+# Contributing to ANTLR 4
+
+1. [Fork](https://help.github.com/articles/fork-a-repo) the [antlr/antlr4 repo](https://github.com/antlr/antlr4)
+2. Install and configure [EditorConfig](http://editorconfig.org/) so your text editor or IDE uses the ANTLR 4 coding style
+3. [Build ANTLR 4](doc/building-antlr.md)
+4. [Run the ANTLR project unit tests](doc/antlr-project-testing.md)
+5. Create a [pull request](https://help.github.com/articles/using-pull-requests/) including your change
+
+
+**Note:** You must sign the `contributors.txt` certificate of origin with your pull request if you've not done so before.

LICENSE.txt

@@ -24,3 +24,29 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+=====
+
+MIT License for codepointat.js from https://git.io/codepointat
+MIT License for fromcodepoint.js from https://git.io/vDW1m
+
+Copyright Mathias Bynens <https://mathiasbynens.be/>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

antlr4-maven-plugin/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>org.antlr</groupId>
     <artifactId>antlr4-master</artifactId>
-    <version>4.6</version>
+    <version>4.6.1-SNAPSHOT</version>
   </parent>
   <artifactId>antlr4-maven-plugin</artifactId>
   <packaging>maven-plugin</packaging>

antlr4-maven-plugin/src/main/java/org/antlr/mojo/antlr4/Antlr4Mojo.java

@@ -150,6 +150,13 @@ public class Antlr4Mojo extends AbstractMojo {
     @Parameter(property = "project", required = true, readonly = true)
     protected MavenProject project;
 
+	/**
+	 * Specifies whether sources are added to the {@code compile} or
+	 * {@code test} scope.
+	 */
+	@Parameter(property = "antlr4.generateTestSources", defaultValue = "false")
+	private boolean generateTestSources;
+
     /**
      * The directory where the ANTLR grammar files ({@code *.g4}) are located.
      */
@@ -190,7 +197,12 @@ public File getLibDirectory() {
     }
 
     void addSourceRoot(File outputDir) {
-        project.addCompileSourceRoot(outputDir.getPath());
+		if (generateTestSources) {
+			project.addTestCompileSourceRoot(outputDir.getPath());
+		}
+		else {
+			project.addCompileSourceRoot(outputDir.getPath());
+		}
     }
 
     /**

contributors.txt

@@ -127,4 +127,14 @@ YYYY/MM/DD, github id, Full name, email
 2016/12/01, samtatasurya, Samuel Tatasurya, [email protected]
 2016/12/03, redxdev, Samuel Bloomberg, [email protected]
 2016/12/11, Gaulouis, Gaulouis, [email protected]
+2016/12/22, akosthekiss, Akos Kiss, [email protected]
+2016/12/24, adrpo, Adrian Pop, [email protected]
+2017/01/11, robertbrignull, Robert Brignull, [email protected]
+2017/01/13, marcelo-rocha, Marcelo Rocha, [email protected]
+2017/01/23, bhamiltoncx, Ben Hamilton, [email protected]
+2017/01/18, mshockwave, Bekket McClane, [email protected]
+2017/02/10, lionelplessis, Lionel Plessis, [email protected]
+2017/02/14, lecode-official, David Neumann, [email protected]
+2017/02/14, xied75, Dong Xie, [email protected]
 2017/02/20, Thomasb81, Thomas Burg, [email protected]
+

doc/ace-javascript-target.md

@@ -174,13 +174,20 @@ At this point, you should have an editor which displays an error icon next to th
 
 What remains to be done is have our validate function actually validate the input. Finally ANTLR comes in the picture!
 
-To start with, let's load ANTLR and your parser, listener etc.. Easy, since you could write:
+To start with, let's load ANTLR and your parser, listener etc.. 
+
+The preferred approach for loading parser code is to bundle your parser, [as described here](javascript-target.md).
+You can then load it as part of the importScripts instruction at the start of your worker code.
+
+Another approach is to load it using 'require'. Easy, since you could write:
 
 ```js
 var antlr4 = require('antlr4/index');
 ```
 
-This may work, but it's actually unreliable. The reason is that the require function used by ANTLR, which exactly mimics the NodeJS require function, uses a different syntax than the require function that comes with ACE. So we need to bring in a require function that conforms to the NodeJS syntax. I personally use one that comes from Torben Haase's Honey project, which you can find here. But hey, now we're going to have 2 'require' functions not compatible with each other! Indeed, this is why you need to take special care, as follows:
+This may work, but it's actually unreliable. The reason is that the 'require' function that comes with ACE uses a different syntax than the 'require' function used by ANTLR, which follows the NodeJS 'require' convention. 
+So we need to bring in a NodeJS compatible 'require' function that conforms to the NodeJS syntax. I personally use one that comes from Torben Haase's Honey project, which you can find in li/require.js. 
+But hey, now we're going to have 2 'require' functions not compatible with each other! Indeed, this is why you need to take special care, as follows:
 
 ```js
 // load nodejs compatible require
@@ -191,7 +198,8 @@ importScripts("../lib/require.js");
 var antlr4_require = require;
 require = ace_require;
 ```
-Now it's safe to load antlr, and the parsers generated for your language. Assuming that your language files (generated or hand-built) are in a folder with an index.js file that calls require for each file, your parser loading code can be as simple as follows:
+Now it's safe to load antlr and the parsers generated for your language. 
+Assuming that your language files (generated or hand-built) are in a folder with an index.js file that calls require for each file, your parser loading code can be as simple as follows:
 ```js
 // load antlr4 and myLanguage
 var antlr4, mylanguage;
@@ -204,6 +212,7 @@ try {
 }
 ```
 Please note the try-finally construct. ANTLR uses 'require' synchronously so it's perfectly safe to ignore the ACE 'require' while running ANTLR code. ACE itself does not guarantee synchronous execution, so you are much safer always switching 'require' back to 'ace_require'.
+
 Now detecting deep syntax errors in your code is a task for your ANTLR listener or visitor or whatever piece of code you've delegated this to. We're not going to describe this here, since it would require some knowledge of your language. However, detecting grammar syntax errors is something ANTLR does beautifully (isn't that why you went for ANTLR in the first place?). So what we will illustrate here is how to report grammar syntax errors. I have no doubt that from there, you will be able to extend the validator to suit your specific needs.
 Whenever ANTLR encounters an unexpected token, it fires an error. By default, the error is routed to an error listener which simply writes to the console.
 What we need to do is replace this listener by our own listener, se we can route errors to the ACE editor. First, let's create such a listener:
@@ -243,5 +252,4 @@ var validate = function(input) {
 };
 ```
 You know what? That's it! You now have an ACE editor that does syntax validation using ANTLR! I hope you find this useful, and simple enough to get started.
-What I did not address here is packaging, not something I'm an expert at. The good news is that it makes development simple, since I don't have to run any compilation process. I just edit my code, reload my editor page, and check how it goes.
-Now wait, hey! How do you debug this? Well, as usual, using Chrome, since neither Firefox or Safari are able to debug worker code. What a shame...
+WNow wait, hey! How do you debug this? Well, as usual, using Chrome, since no other browser is able to debug worker code. What a shame...

doc/cpp-target.md

@@ -79,15 +79,10 @@ This example assumes your grammar contains a parser rule named `key` for which t
 There are a couple of things that only the C++ ANTLR target has to deal with. They are described here.
 
 ### Build Aspects
-
-The code generation (by running the ANTLR4 jar) allows to specify 2 values you
-might find useful for better integration of the generated files into your
-application (both are optional):
+The code generation (by running the ANTLR4 jar) allows to specify 2 values you might find useful for better integration of the generated files into your application (both are optional):
 
 * A **namespace**: use the **`-package`** parameter to specify the namespace you want.
-* An **export macro**: especially in VC++ extra work is required to export your classes from a DLL. 
-This is usually accomplished by a macro that has different values depending on whether 
-you are creating the DLL or import it. The ANTLR4 runtime itself also uses one for its classes:
+* An **export macro**: especially in VC++ extra work is required to export your classes from a DLL. This is usually accomplished by a macro that has different values depending on whether you are creating the DLL or import it. The ANTLR4 runtime itself also uses one for its classes:
 
 ```c++
   #ifdef ANTLR4CPP_EXPORTS
@@ -100,30 +95,17 @@ you are creating the DLL or import it. The ANTLR4 runtime itself also uses one f
     #endif
   #endif
 ```
-
-Just like the `ANTLR4CPP_PUBLIC` macro here you can specify your own one for
-the generated classes using the **`-DexportMacro=...`** command-line parameter or
+Just like the `ANTLR4CPP_PUBLIC` macro here you can specify your own one for the generated classes using the **`-DexportMacro=...`** command-line parameter or
 grammar option `options {exportMacro='...';}` in your grammar file.
 
-In order to create a static lib in Visual Studio define the `ANTLR4CPP_STATIC` macro
-in addition to the project settings that must be set for a static library
-(if you compile the runtime yourself).
+In order to create a static lib in Visual Studio define the `ANTLR4CPP_STATIC` macro in addition to the project settings that must be set for a static library (if you compile the runtime yourself).
 
-For gcc and clang it is possible to use the `-fvisibility=hidden` setting to
-hide all symbols except those that are made default-visible (which has been 
-defined for all public classes in the runtime).
+For gcc and clang it is possible to use the `-fvisibility=hidden` setting to hide all symbols except those that are made default-visible (which has been defined for all public classes in the runtime).
 
 ### Memory Management
-
-Since C++ has no built-in memory management we need to take extra care.
-For that we rely mostly on smart pointers, which however might cause time 
-penalties or memory side effects (like cyclic references) if not used with care.
-Currently however the memory household looks very stable. Generally, when you
-see a raw pointer in code consider this as being managed elsewehere. You
-should never try to manage such a pointer (delete, assign to smart pointer etc.).
+Since C++ has no built-in memory management we need to take extra care. For that we rely mostly on smart pointers, which however might cause time penalties or memory side effects (like cyclic references) if not used with care. Currently however the memory household looks very stable. Generally, when you see a raw pointer in code consider this as being managed elsewehere. You should never try to manage such a pointer (delete, assign to smart pointer etc.).
 
 ### Unicode Support
-
 Encoding is mostly an input issue, i.e. when the lexer converts text input into lexer tokens. The parser is completely encoding unaware. However, lexer input in the grammar is defined by character ranges with either a single member (e.g. 'a' or [a] or [abc]), an explicit range (e.g. 'a'..'z' or [a-z]), the full Unicode range (for a wildcard) and the full Unicode range minus a sub range (for negated ranges, e.g. ~[a]). The explicit ranges (including single member ranges) are encoded in the serialized ATN by 16bit numbers, hence cannot reach beyond 0xFFFF (the Unicode BMP), while the implicit ranges can include any value (and hence support the full Unicode set, up to 0x10FFFF).
 
 > An interesting side note here is that the Java target fully supports Unicode as well, despite the inherent limitations from the serialized ATN. That's possible because the Java String class represents characters beyond the BMP as surrogate pairs (two 16bit values) and even reads them as 2 separate input characters. To make this work a character range for an identifier in a grammar must include the surrogate pairs area (for a Java parser).
@@ -133,7 +115,6 @@ The C++ target however always expects UTF-8 input (either in a string or via a w
 The differences in handling characters beyond the BMP leads to a difference between Java and C++ lexers: the character offsets may not concur. This is because Java reads two 16bit values per Unicode char (if that falls into the surrogate area) while a C++ parser only reads one 32bit value. That usually doesn't have practical consequences, but might confuse people when comparing token positions.
 
 ### Named Actions
-
 In order to help customizing the generated files there are a number of additional socalled **named actions**. These actions are tight to specific areas in the generated code and allow to add custom (target specific) code. All targets support these actions 
 
 * @parser::header

doc/index.md

@@ -16,7 +16,7 @@ Links in the documentation refer to various sections of the book but have been r
 <a href=""><img src=images/tpantlr2.png width=120></a>
 <a href=""><img src=images/tpdsl.png width=120></a>
 <a href="https://www.youtube.com/watch?v=OAoA3E-cyug"><img src=images/teronbook.png width=250></a>
-                 
+
 This documentation is a reference and summarizes grammar syntax and the key semantics of ANTLR grammars. The source code for all examples in the book, not just this chapter, are free at the publisher's website. The following video is a general tour of ANTLR 4 and includes a description of how to use parse tree listeners to process Java files easily:
 
 <a href="https://vimeo.com/59285751"><img src=images/tertalk.png width=200></a>
@@ -52,7 +52,7 @@ This documentation is a reference and summarizes grammar syntax and the key sema
 * [Runtime Libraries and Code Generation Targets](targets.md)
 
 * [Parsing binary streams](parsing-binary-files.md)
- 
+
 * [Parser and lexer interpreters](interpreters.md)
 
 * [Resources](resources.md)
@@ -61,6 +61,8 @@ This documentation is a reference and summarizes grammar syntax and the key sema
 
 * [Building ANTLR itself](building-antlr.md)
 
+* [Contributing to ANTLR](/CONTRIBUTING.md)
+
 * [Cutting an ANTLR Release](releasing-antlr.md)
 
 * [ANTLR project unit tests](antlr-project-testing.md)