Add tests for handling empty lines

ballerina/tests/parse_string.bal

@@ -745,6 +745,18 @@ isolated function testBlockScalarValue() returns error? {
     test:assertEquals(value, expectedResult);
 }
 
+@test:Config
+isolated function testParsingEmptyLines() returns error? {
+    string content = check io:fileReadString(FILE_PATH + "nested_20.yaml");
+    map<string> value = check parseString(content);
+    map<string> expectedResult = {
+        "FoldingDoubleQuote": "Empty line\nas a line feed",
+        "FoldingSingleQuote": "Empty line\nas a line feed",
+        "Chomping": "Clipped empty lines\n"
+    };
+    test:assertEquals(value, expectedResult);
+}
+
 @test:Config
 isolated function testByteAsExpectedTypeForParseString() returns error? {
     byte result = check parseString("1");

ballerina/tests/resources/nested_20.yaml

@@ -0,0 +1,10 @@
+FoldingDoubleQuote:
+  "Empty line
+  
+  as a line feed"
+FoldingSingleQuote:
+  'Empty line
+  
+  as a line feed'
+Chomping: |
+  Clipped empty lines

native/src/main/java/io/ballerina/lib/data/yaml/lexer/LexerState.java

@@ -83,7 +83,7 @@ public class LexerState {
     public static final State LEXER_DOUBLE_QUOTE = new DoubleQuoteState();
     public static final State LEXER_SINGLE_QUOTE = new SingleQuoteState();
     public static final State LEXER_BLOCK_HEADER = new BlockHeaderState();
-    public static final State LEXER_LITERAL = new LiteralState();
+    public static final State LEXER_BLOCK_SCALAR = new BlockScalarState();
     public static final State LEXER_RESERVED_DIRECTIVE = new ReservedDirectiveState();
     private State state = LEXER_START_STATE;
     private final CharacterReader characterReader;
@@ -764,6 +764,10 @@ public State transition(LexerState lexerState) throws Error.YamlParserException
                 }
             }
 
+            if (Scanner.scanAndTokenizeEOL(lexerState, EMPTY_LINE)) {
+                return this;
+            }
+
             // Terminating delimiter
             if (lexerState.peek() == '\"') {
                 IndentUtils.handleMappingValueIndent(lexerState, DOUBLE_QUOTE_DELIMITER);
@@ -809,6 +813,10 @@ public State transition(LexerState lexerState) throws Error.YamlParserException
                 }
             }
 
+            if (Scanner.scanAndTokenizeEOL(lexerState, EMPTY_LINE)) {
+                return this;
+            }
+
             // Escaped single quote
             if (lexerState.peek() == '\'' && lexerState.peek(1) == '\'') {
                 lexerState.lexeme += "'";
@@ -878,7 +886,7 @@ public State transition(LexerState lexerState) throws Error.YamlParserException
         }
     }
 
-    private static class LiteralState implements State {
+    private static class BlockScalarState implements State {
 
         /**
          * Scan the lexemes for block scalar.
@@ -906,6 +914,10 @@ public State transition(LexerState lexerState) throws Error.YamlParserException
                     return LexerState.LEXER_START_STATE.transition(lexerState);
                 }
 
+                if (Scanner.scanAndTokenizeEOL(lexerState, EMPTY_LINE)) {
+                    return this;
+                }
+
                 switch (lexerState.peek()) {
                     case '#' -> { // Generate beginning of the trailing comment
                         if (!lexerState.trailingComment && lexerState.captureIndent) {
@@ -928,11 +940,6 @@ public State transition(LexerState lexerState) throws Error.YamlParserException
                     case ':', '-' -> {
                         return LexerState.LEXER_START_STATE.transition(lexerState);
                     }
-                    case -1 -> { // Empty lines are allowed in trailing comments
-                        lexerState.forward();
-                        lexerState.tokenize(EMPTY_LINE);
-                        return this;
-                    }
                     default -> { // Other characters are not allowed when the indentation is less
                         throw new Error.YamlParserException("insufficient indent to process literal characters",
                                 lexerState.getLine(), lexerState.getColumn());
@@ -942,6 +949,10 @@ public State transition(LexerState lexerState) throws Error.YamlParserException
 
             if (lexerState.trailingComment) {
                 while (true) {
+                    if (Scanner.scanAndTokenizeEOL(lexerState, EMPTY_LINE)) {
+                        return this;
+                    }
+
                     switch (lexerState.peek()) {
                         case ' ' -> { // Ignore whitespace
                             lexerState.forward();
@@ -950,11 +961,6 @@ public State transition(LexerState lexerState) throws Error.YamlParserException
                             lexerState.tokenize(EOL);
                             return this;
                         }
-                        case -1 -> { // Empty lines are allowed in trailing comments
-                            lexerState.forward();
-                            lexerState.tokenize(EMPTY_LINE);
-                            return this;
-                        }
                         default -> {
                             throw new Error.YamlParserException("invalid trailing comment",
                                     lexerState.getLine(), lexerState.getColumn());

native/src/main/java/io/ballerina/lib/data/yaml/lexer/Scanner.java

@@ -141,7 +141,7 @@ public static class SingleQuoteCharScanner implements Scan {
          * Process double quoted scalar values.
          */
         @Override
-        public boolean scan(LexerState sm) {
+        public boolean scan(LexerState sm) throws Error.YamlParserException {
             // Process nb-json characters
             if (matchPattern(sm, List.of(JSON_PATTERN), List.of(new Utils.CharPattern('\'')))) {
                 sm.appendToLexeme(Character.toString(sm.peek()));
@@ -155,9 +155,14 @@ public boolean scan(LexerState sm) {
                     sm.forward();
                     return false;
                 }
+                return true;
             }
 
-            return true;
+            if (Utils.isNewLine(sm)) {
+                return true;
+            }
+
+            throw new Error.YamlParserException("invalid character", sm.getLine(), sm.getColumn());
         }
     }
 
@@ -185,7 +190,15 @@ public boolean scan(LexerState sm) throws Error.YamlParserException {
                 return false;
             }
 
-            return true;
+            if (sm.peek() == '\"') {
+                return true;
+            }
+
+            if (Utils.isNewLine(sm)) {
+                return true;
+            }
+
+            throw new Error.YamlParserException("invalid character", sm.getLine(), sm.getColumn());
         }
     }
 

native/src/main/java/io/ballerina/lib/data/yaml/lexer/Utils.java

@@ -253,4 +253,8 @@ public static String getWhitespace(LexerState sm) {
         }
         return whitespace.toString();
     }
+
+    public static boolean isNewLine(LexerState sm) {
+        return sm.peek() == '\n' || sm.peek() == '\r' && sm.peek(1) == '\n';
+    }
 }

native/src/main/java/io/ballerina/lib/data/yaml/lexer/YamlLexer.java

@@ -74,14 +74,12 @@ public static LexerState.State scanTokens(LexerState state) throws Error.YamlPar
             return state.getState();
         }
 
-        // Check for line breaks when reading from string
         if (state.peek() == '\n' && state.getState() != LEXER_DOUBLE_QUOTE) {
             state.forward();
             state.tokenize(EOL);
             return state.getState();
         }
 
-        // Check for line breaks when reading from string
         if (state.peek() == '\r' && state.peek(1) == '\n' && state.getState() != LEXER_DOUBLE_QUOTE) {
             state.forward();
             state.tokenize(EOL);

native/src/main/java/io/ballerina/lib/data/yaml/parser/YamlParser.java

@@ -77,13 +77,11 @@
 import static io.ballerina.lib.data.yaml.lexer.Token.TokenType.EMPTY_LINE;
 import static io.ballerina.lib.data.yaml.lexer.Token.TokenType.EOL;
 import static io.ballerina.lib.data.yaml.lexer.Token.TokenType.FOLDED;
-import static io.ballerina.lib.data.yaml.lexer.Token.TokenType.MAPPING_END;
 import static io.ballerina.lib.data.yaml.lexer.Token.TokenType.MAPPING_KEY;
 import static io.ballerina.lib.data.yaml.lexer.Token.TokenType.MAPPING_VALUE;
 import static io.ballerina.lib.data.yaml.lexer.Token.TokenType.PLANAR_CHAR;
 import static io.ballerina.lib.data.yaml.lexer.Token.TokenType.SEPARATION_IN_LINE;
 import static io.ballerina.lib.data.yaml.lexer.Token.TokenType.SEPARATOR;
-import static io.ballerina.lib.data.yaml.lexer.Token.TokenType.SEQUENCE_END;
 import static io.ballerina.lib.data.yaml.lexer.Token.TokenType.SEQUENCE_ENTRY;
 import static io.ballerina.lib.data.yaml.lexer.Token.TokenType.SINGLE_QUOTE_DELIMITER;
 import static io.ballerina.lib.data.yaml.lexer.Token.TokenType.TAG;
@@ -1345,7 +1343,7 @@ private static YamlEvent parse(ParserState state, ParserUtils.ParserOption optio
                 return new YamlEvent.EndEvent(Collection.MAPPING);
             }
             case LITERAL, FOLDED -> {
-                state.updateLexerState(LexerState.LEXER_LITERAL);
+                state.updateLexerState(LexerState.LEXER_BLOCK_SCALAR);
                 return appendData(state, option, true);
             }
         }
@@ -1909,7 +1907,7 @@ private static String blockScalar(ParserState state, boolean isFolded) throws Er
             }
         }
 
-        state.getLexerState().updateLexerState(LexerState.LEXER_LITERAL);
+        state.getLexerState().updateLexerState(LexerState.LEXER_BLOCK_SCALAR);
         StringBuilder lexemeBuffer = new StringBuilder();
         StringBuilder newLineBuffer = new StringBuilder();
         boolean isFirstLine = true;
@@ -2185,10 +2183,8 @@ private static String doubleQuoteScalar(ParserState state) throws Error.YamlPars
                     }
                     state.getLexerState().setFirstLine(false);
                 }
-                default -> {
-                    throw new Error.YamlParserException("invalid double quote scalar",
-                            state.getLine(), state.getColumn());
-                }
+                default -> throw new Error.YamlParserException("invalid double quote scalar",
+                        state.getLine(), state.getColumn());
             }
             getNextToken(state);
         }
@@ -2283,10 +2279,8 @@ private static String singleQuoteScalar(ParserState state) throws Error.YamlPars
                     }
                     state.getLexerState().setFirstLine(false);
                 }
-                default -> {
-                    throw new Error.YamlParserException("invalid single quote character",
-                            state.getLine(), state.getColumn());
-                }
+                default -> throw new Error.YamlParserException("invalid single quote character",
+                        state.getLine(), state.getColumn());
             }
             getNextToken(state);
         }