Add ECMAScript test suite (test262) (#567)

.github/workflows/test262.yml

@@ -0,0 +1,69 @@
+name: EcmaScript official test suite (test262)
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - v*
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  run_test262:
+    name: Run the test262 test suite
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+      - name: Install the Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+          profile: minimal
+      - name: Cache cargo registry
+        uses: actions/cache@v1
+        with:
+          path: ~/.cargo/registry
+          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
+      - name: Cache cargo index
+        uses: actions/cache@v1
+        with:
+          path: ~/.cargo/git
+          key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+      - name: Cache cargo build
+        uses: actions/cache@v1
+        with:
+          path: target
+          key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }}
+
+      # Run the test suite and upload the results
+      - name: Checkout GitHub pages
+        if: github.event_name == 'push'
+        uses: actions/checkout@v2
+        with:
+          ref: gh-pages
+          path: gh-pages
+      - run: mkdir -p gh-pages/test262
+
+      - name: Run the test262 test suite
+        run: cargo run --release --bin boa_tester -- -o gh-pages/test262
+
+      - name: Commit files
+        if: github.event_name == 'push'
+        run: |
+          cd gh-pages
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git pull
+          git commit -m "Add new test262 results" -a
+          cd ..
+      - name: Upload results
+        if: github.event_name == 'push'
+        uses: ad-m/[email protected]
+        with:
+          directory: gh-pages
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/webassembly.yml

@@ -16,10 +16,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Check code formatting
-        uses: creyD/[email protected]
-        with:
-          dry: true
-          prettier_options: --check .
+        run: npx prettier --check .
 
   build:
     name: Build webassembly demo
@@ -34,7 +31,7 @@ jobs:
           override: true
           profile: minimal
       - name: Install wasm-pack
-        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh 
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
       - name: Cache yarn build
         uses: actions/cache@v2
         with:

.gitignore

@@ -23,4 +23,7 @@ tests/js/test.js
 *.string_data
 *.string_index
 *.events
-chrome_profiler.json 
+chrome_profiler.json
+
+# Logs
+*.log

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test262"]
+	path = test262
+	url = https://github.com/tc39/test262.git

.prettierignore

@@ -6,3 +6,5 @@ boa/benches/bench_scripts/mini_js.js
 boa/benches/bench_scripts/clean_js.js
 boa_wasm/pkg
 dist
+test262
+tests/js/test.js

CONTRIBUTING.md

@@ -65,6 +65,25 @@ There are some pre-defined tasks in [tasks.json](.vscode/tasks.json)
 If you don't want to install everything on your machine, you can use the Dockerfile.
 Start VSCode in container mode (you may need the docker container plugin) and use the Dockerfile.
 
+## Testing
+
+Boa provides its own test suite, and can also run the official ECMAScript test suite. To run the Boa test
+suite, you can just run the normal `cargo test`, and to run the full ECMAScript test suite, you can run it
+with this command:
+
+```
+cargo run --release --bin boa_tester -- -v 2> error.log
+```
+
+Note that this requires the `test262` submodule to be checked out, so you will need to run the following first:
+
+```
+git submodule init && git submodule update
+```
+
+This will run the test suite in verbose mode (you can remove the `-- -v` part to run it in non-verbose mode),
+and output nice colorings in the terminal. It will also output any panic information into the `error.log` file.
+
 ## Communication
 
 We have a Discord server, feel free to ask questions here:

Cargo.toml

@@ -3,6 +3,7 @@ members = [
     "boa",
     "boa_cli",
     "boa_wasm",
+    "tester",
 ]
 
 # The release profile, used for `cargo build --release`.

boa/src/builtins/array/mod.rs

@@ -655,7 +655,10 @@ impl Array {
         }
 
         let search_element = args[0].clone();
-        let len = this.get_field("length").as_number().unwrap() as i32;
+        let len = this
+            .get_field("length")
+            .as_number()
+            .expect("length was not a number") as i32;
 
         let mut idx = match args.get(1) {
             Some(from_idx_ptr) => {

boa/src/builtins/math/tests.rs

@@ -721,7 +721,7 @@ fn tan() {
     assert!(float_cmp::approx_eq!(
         f64,
         a.to_number(&mut engine).unwrap(),
-        f64::from(1.964_759_657_248_652_5)
+        1.964_759_657_248_652_5
     ));
 }
 

boa/src/exec/tests.rs

@@ -1303,7 +1303,7 @@ fn assignment_to_non_assignable() {
     for case in test_cases.iter() {
         let string = forward(&mut engine, case);
 
-        assert!(string.starts_with("Syntax Error: "));
+        assert!(string.starts_with("Uncaught \"SyntaxError\": "));
         assert!(string.contains("1:3"));
     }
 }
@@ -1318,9 +1318,9 @@ fn multicharacter_assignment_to_non_assignable() {
     let test_cases = ["3 **= 5", "3 <<= 5", "3 >>= 5"];
 
     for case in test_cases.iter() {
-        let string = forward(&mut engine, case);
+        let string = dbg!(forward(&mut engine, case));
 
-        assert!(string.starts_with("Syntax Error: "));
+        assert!(string.starts_with("Uncaught \"SyntaxError\": "));
         assert!(string.contains("1:3"));
     }
 }
@@ -1335,9 +1335,9 @@ fn multicharacter_bitwise_assignment_to_non_assignable() {
     let test_cases = ["3 >>>= 5", "3 &&= 5", "3 ||= 5", "3 ??= 5"];
 
     for case in test_cases.iter() {
-        let string = forward(&mut engine, case);
+        let string = dbg!(forward(&mut engine, case));
 
-        assert!(string.starts_with("Syntax Error: "));
+        assert!(string.starts_with("Uncaught \"SyntaxError\": "));
         assert!(string.contains("1:3"));
     }
 }
@@ -1347,10 +1347,10 @@ fn assign_to_array_decl() {
     let realm = Realm::create();
     let mut engine = Interpreter::new(realm);
 
-    assert!(forward(&mut engine, "[1] = [2]").starts_with("Syntax Error: "));
-    assert!(forward(&mut engine, "[3, 5] = [7, 8]").starts_with("Syntax Error: "));
-    assert!(forward(&mut engine, "[6, 8] = [2]").starts_with("Syntax Error: "));
-    assert!(forward(&mut engine, "[6] = [2, 9]").starts_with("Syntax Error: "));
+    assert!(forward(&mut engine, "[1] = [2]").starts_with("Uncaught \"SyntaxError\": "));
+    assert!(forward(&mut engine, "[3, 5] = [7, 8]").starts_with("Uncaught \"SyntaxError\": "));
+    assert!(forward(&mut engine, "[6, 8] = [2]").starts_with("Uncaught \"SyntaxError\": "));
+    assert!(forward(&mut engine, "[6] = [2, 9]").starts_with("Uncaught \"SyntaxError\": "));
 }
 
 #[test]
@@ -1359,7 +1359,7 @@ fn assign_to_object_decl() {
     let mut engine = Interpreter::new(realm);
 
     const ERR_MSG: &str =
-        "expected token \';\', got \':\' in expression statement at line 1, col 3";
+        "Uncaught \"SyntaxError\": \"expected token \';\', got \':\' in expression statement at line 1, col 3\"";
 
     assert_eq!(forward(&mut engine, "{a: 3} = {a: 5};"), ERR_MSG);
 }

boa/src/lib.rs

@@ -60,22 +60,33 @@ pub use gc::{custom_trace, unsafe_empty_trace, Finalize, Trace};
 #[must_use]
 pub type Result<T> = StdResult<T, Value>;
 
-fn parser_expr(src: &str) -> StdResult<StatementList, String> {
-    Parser::new(src.as_bytes())
-        .parse_all()
-        .map_err(|e| e.to_string())
+/// Parses the given source code.
+///
+/// It will return either the statement list AST node for the code, or a parsing error if something
+/// goes wrong.
+#[inline]
+pub fn parse(src: &str) -> StdResult<StatementList, ParseError> {
+    Parser::new(src.as_bytes()).parse_all()
 }
 
 /// Execute the code using an existing Interpreter
 /// The str is consumed and the state of the Interpreter is changed
 pub fn forward(engine: &mut Interpreter, src: &str) -> String {
     // Setup executor
-    let expr = match parser_expr(src) {
+    let expr = match parse(src) {
         Ok(res) => res,
-        Err(e) => return e,
+        Err(e) => {
+            return format!(
+                "Uncaught {}",
+                engine
+                    .throw_syntax_error(e.to_string())
+                    .expect_err("interpreter.throw_syntax_error() did not return an error")
+                    .display()
+            );
+        }
     };
     expr.run(engine).map_or_else(
-        |e| format!("Error: {}", e.display()),
+        |e| format!("Uncaught {}", e.display()),
         |v| v.display().to_string(),
     )
 }
@@ -88,13 +99,13 @@ pub fn forward(engine: &mut Interpreter, src: &str) -> String {
 pub fn forward_val(engine: &mut Interpreter, src: &str) -> Result<Value> {
     let main_timer = BoaProfiler::global().start_event("Main", "Main");
     // Setup executor
-    let result = match parser_expr(src) {
-        Ok(expr) => expr.run(engine),
-        Err(e) => {
-            eprintln!("{}", e);
-            panic!();
-        }
-    };
+    let result = parse(src)
+        .map_err(|e| {
+            engine
+                .throw_syntax_error(e.to_string())
+                .expect_err("interpreter.throw_syntax_error() did not return an error")
+        })
+        .and_then(|expr| expr.run(engine));
 
     // The main_timer needs to be dropped before the BoaProfiler is.
     drop(main_timer);

boa/src/syntax/ast/position.rs

@@ -26,6 +26,7 @@ pub struct Position {
 impl Position {
     /// Creates a new `Position`.
     #[inline]
+    #[track_caller]
     pub fn new(line_number: u32, column_number: u32) -> Self {
         Self {
             line_number: NonZeroU32::new(line_number).expect("line number cannot be 0"),
@@ -65,6 +66,7 @@ pub struct Span {
 impl Span {
     /// Creates a new `Span`.
     #[inline]
+    #[track_caller]
     pub fn new(start: Position, end: Position) -> Self {
         assert!(start <= end, "a span cannot start after its end");