From 5e1b313917638cd4b3a1732ac1d7e6f35fc7f434 Mon Sep 17 00:00:00 2001
From: Simon Lydell <simon.lydell@gmail.com>
Date: Sat, 17 Aug 2024 22:45:20 +0200
Subject: [PATCH] Add tests and docs for ES2024
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The only syntax change is the `v` flag for regular expressions. However,
the spec didn't change anything in the tokenization step – only in later
regexp parsing steps, so no changes were needed in js-tokens.
---
 README.md          | 7 +++++--
 test/index.test.js | 5 +++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9e0dbd2..2d0afec 100644
--- a/README.md
+++ b/README.md
@@ -129,7 +129,9 @@ Just like for StringLiteral, templates can also contain invalid escapes. `` `\u`
 
 _Spec: [RegularExpressionLiteral]_
 
-Regex literals may contain invalid regex syntax. They are still matched as regex literals.
+Regex literals may contain invalid regex syntax. They are still matched as regex literals. The specification even says:
+
+> The productions below [...] are used by the input element scanner to find the end of the regular expression literal. The source text [...] are subsequently parsed again using the more stringent ECMAScript Regular Expression grammar.
 
 If the ending `/` is missing, the token has `closed: false`. JavaScript regex literals cannot contain newlines (not even escaped ones), so unclosed regex literals simply end at the end of the line.
 
@@ -146,6 +148,7 @@ Examples:
 /a/Inva1id
 /+/
 /[/]\//
+/[\p{Decimal_Number}--[0-9]]/v
 ```
 
 ### MultiLineComment
@@ -434,7 +437,7 @@ All possible values in JSX children:
 
 The intention is to always support the latest ECMAScript version whose feature set has been finalized.
 
-Currently, ECMAScript 2023 is supported.
+Currently, ECMAScript 2024 is supported.
 
 #### Annex B and C (strict mode)
 
diff --git a/test/index.test.js b/test/index.test.js
index bab0854..c2e9ea1 100644
--- a/test/index.test.js
+++ b/test/index.test.js
@@ -815,6 +815,11 @@ describe("Token", () => {
     match("/?foo/");
     match("/*foo/", false);
 
+    match("/[p{Decimal_Number}--[0-9]]/v");
+    match("/[[[a-z]--[b-d]]&&[0-2[q{a|bc|def}]]]/v");
+    match("/[/]/v"); // Valid token, invalid at later regex parsing stage.
+    match("/[[a]/]/v", "/[[a]/"); // This is why slash isn't valid in character class when nesting came to play.
+
     match("/a/d");
     match("/a/g");
     match("/a/m");