bpo-43822: Improve syntax errors for missing commas (pythonGH-25377)

Grammar/Tokens

@@ -59,6 +59,7 @@ AWAIT
 ASYNC
 TYPE_IGNORE
 TYPE_COMMENT
+SOFT_KEYWORD
 ERRORTOKEN
 
 # These aren't used by the C tokenizer but are needed for tokenize.py

Grammar/python.gram

@@ -7,6 +7,7 @@ _PyPegen_parse(Parser *p)
     // Initialize keywords
     p->keywords = reserved_keywords;
     p->n_keyword_lists = n_keyword_lists;
+    p->soft_keywords = soft_keywords;
 
     // Run parser
     void *result = NULL;
@@ -459,6 +460,7 @@ expressions[expr_ty]:
     | a=expression ',' { _PyAST_Tuple(CHECK(asdl_expr_seq*, _PyPegen_singleton_seq(p, a)), Load, EXTRA) }
     | expression
 expression[expr_ty] (memo):
+    | invalid_expression
     | a=disjunction 'if' b=disjunction 'else' c=expression { _PyAST_IfExp(b, a, c, EXTRA) }
     | disjunction
     | lambdef
@@ -778,6 +780,13 @@ invalid_kwarg:
     | expression a='=' {
         RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
             a, "expression cannot contain assignment, perhaps you meant \"==\"?") }
+
+invalid_expression:
+    # !(NAME STRING) is not matched so we don't show this error with some invalid string prefixes like: kf"dsfsdf"
+    # Soft keywords need to also be ignored because they can be parsed as NAME NAME
+    | !(NAME STRING | SOFT_KEYWORD) a=disjunction expression {
+        RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->end_col_offset - 1, "invalid syntax. Perhaps you forgot a comma?") }
+
 invalid_named_expression:
     | a=expression ':=' expression {
         RAISE_SYNTAX_ERROR_KNOWN_LOCATION(

Lib/test/test_genexps.py

@@ -103,7 +103,7 @@
     >>> dict(a = i for i in range(10))
     Traceback (most recent call last):
        ...
-    SyntaxError: invalid syntax
+    SyntaxError: invalid syntax. Maybe you meant '==' or ':=' instead of '='?
 
 Verify that parenthesis are required when used as a keyword argument value
 

Lib/test/test_syntax.py

@@ -248,22 +248,36 @@
 
 # Missing commas in literals collections should not
 # produce special error messages regarding missing
-# parentheses
+# parentheses, but about missing commas instead
 
 >>> [1, 2 3]
 Traceback (most recent call last):
-SyntaxError: invalid syntax
+SyntaxError: invalid syntax. Perhaps you forgot a comma?
 
 >>> {1, 2 3}
 Traceback (most recent call last):
-SyntaxError: invalid syntax
+SyntaxError: invalid syntax. Perhaps you forgot a comma?
 
 >>> {1:2, 2:5 3:12}
 Traceback (most recent call last):
-SyntaxError: invalid syntax
+SyntaxError: invalid syntax. Perhaps you forgot a comma?
 
 >>> (1, 2 3)
 Traceback (most recent call last):
+SyntaxError: invalid syntax. Perhaps you forgot a comma?
+
+# Make sure soft keywords constructs don't raise specialized
+# errors regarding missing commas
+
+>>> match x:
+...     y = 3
+Traceback (most recent call last):
+SyntaxError: invalid syntax
+
+>>> match x:
+...     case y:
+...        3 $ 3
+Traceback (most recent call last):
 SyntaxError: invalid syntax
 
 From compiler_complex_args():
@@ -864,7 +878,7 @@
    SyntaxError: cannot assign to attribute here. Maybe you meant '==' instead of '='?
 
 Ensure that early = are not matched by the parser as invalid comparisons
-   >>> f(2, 4, x=34); {1,2 a}
+   >>> f(2, 4, x=34); 1 $ 2
    Traceback (most recent call last):
    SyntaxError: invalid syntax
 

Misc/NEWS.d/next/Core and Builtins/2021-04-13-02-32-18.bpo-43822.lej0OO.rst

@@ -0,0 +1,2 @@
+Improve syntax errors in the parser for missing commas between expressions.
+Patch by Pablo Galindo.