From 0a8d0cc9daace2893186cd777cb0b65301b46896 Mon Sep 17 00:00:00 2001 From: Marc-Alexandre Ghaly Date: Mon, 15 May 2023 15:34:01 -0400 Subject: [PATCH] fixing --- classes/dataflow_lexer.php | 120 ++++++++++++++++++ classes/parser.php | 6 +- .../ExpressionLanguage.php | 3 +- vendor/symfony/expression-language/Lexer.php | 4 - 4 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 classes/dataflow_lexer.php diff --git a/classes/dataflow_lexer.php b/classes/dataflow_lexer.php new file mode 100644 index 00000000..b80d46b1 --- /dev/null +++ b/classes/dataflow_lexer.php @@ -0,0 +1,120 @@ +. + +namespace tool_dataflows; + +use Symfony\Component\ExpressionLanguage\Token; +use Symfony\Component\ExpressionLanguage\SyntaxError; +use Symfony\Component\ExpressionLanguage\TokenStream; + +/** + * Class adding modification to Symfony Lexer to suit dataflow needs. + * + * @package tool_dataflows + * @author Ghaly Marc-Alexandre + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class dataflow_lexer extends \Symfony\Component\ExpressionLanguage\Lexer { + /** + * Tokenizes an expression. + * + * @param string $expression The expression to tokenize + * + * @return TokenStream A token stream instance + * + * @throws SyntaxError + */ + public function tokenize($expression) + { + $expression = str_replace(["\r", "\n", "\t", "\v", "\f"], ' ', $expression); + $cursor = 0; + $tokens = []; + $brackets = []; + $end = \strlen($expression); + + while ($cursor < $end) { + if (' ' == $expression[$cursor]) { + ++$cursor; + + continue; + } + + if (preg_match('/[0-9]+(?:\.[0-9]+)?/A', $expression, $match, 0, $cursor)) { + // Numbers. + // Floats. + $number = (float) $match[0]; + if (preg_match('/^[0-9]+$/', $match[0]) && $number <= \PHP_INT_MAX) { + // Integers lower than the maximum. + $number = (int) $match[0]; + } + $tokens[] = new Token(Token::NUMBER_TYPE, $number, $cursor + 1); + $cursor += \strlen($match[0]); + } elseif (preg_match("/{'(\W[a-zA-Z_\x7f-\xff][a-zA-Z0-9_.\x7f-\xff]*)'}/A", $expression, $match, 0, $cursor)) { + // Names litteral. + $tokens[] = new Token(Token::NAME_TYPE, $match[1], $cursor + 1); + $cursor += \strlen($match[0]); + } elseif (false !== strpos('([{', $expression[$cursor])) { + // Opening bracket. + $brackets[] = [$expression[$cursor], $cursor]; + + $tokens[] = new Token(Token::PUNCTUATION_TYPE, $expression[$cursor], $cursor + 1); + ++$cursor; + } elseif (false !== strpos(')]}', $expression[$cursor])) { + // Closing bracket. + if (empty($brackets)) { + throw new SyntaxError(sprintf('Unexpected "%s".', $expression[$cursor]), $cursor, $expression); + } + + list($expect, $cur) = array_pop($brackets); + if ($expression[$cursor] != strtr($expect, '([{', ')]}')) { + throw new SyntaxError(sprintf('Unclosed "%s".', $expect), $cur, $expression); + } + + $tokens[] = new Token(Token::PUNCTUATION_TYPE, $expression[$cursor], $cursor + 1); + ++$cursor; + } elseif (preg_match('/"([^"\\\\]*(?:\\\\.[^"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'/As', $expression, $match, 0, $cursor)) { + // Strings. + $tokens[] = new Token(Token::STRING_TYPE, stripcslashes(substr($match[0], 1, -1)), $cursor + 1); + $cursor += \strlen($match[0]); + } elseif (preg_match('/(?<=^|[\s(])not in(?=[\s(])|\!\=\=|(?<=^|[\s(])not(?=[\s(])|(?<=^|[\s(])and(?=[\s(])|\=\=\=|\>\=|(?<=^|[\s(])or(?=[\s(])|\<\=|\*\*|\.\.|(?<=^|[\s(])in(?=[\s(])|&&|\|\||(?<=^|[\s(])matches|\=\=|\!\=|\*|~|%|\/|\>|\||\!|\^|&|\+|\<|\-/A', $expression, $match, 0, $cursor)) { + // Operators. + $tokens[] = new Token(Token::OPERATOR_TYPE, $match[0], $cursor + 1); + $cursor += \strlen($match[0]); + } elseif (false !== strpos('.,?:', $expression[$cursor])) { + // Punctuation. + $tokens[] = new Token(Token::PUNCTUATION_TYPE, $expression[$cursor], $cursor + 1); + ++$cursor; + } elseif (preg_match('/[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/A', $expression, $match, 0, $cursor)) { + // Names. + $tokens[] = new Token(Token::NAME_TYPE, $match[0], $cursor + 1); + $cursor += \strlen($match[0]); + } else { + // Unlexable. + throw new SyntaxError(sprintf('Unexpected character "%s".', $expression[$cursor]), $cursor, $expression); + } + } + + $tokens[] = new Token(Token::EOF_TYPE, null, $cursor + 1); + + if (!empty($brackets)) { + list($expect, $cur) = array_pop($brackets); + throw new SyntaxError(sprintf('Unclosed "%s".', $expect), $cur, $expression); + } + + return new TokenStream($tokens, $expression); + } +} \ No newline at end of file diff --git a/classes/parser.php b/classes/parser.php index d9d4d530..1a7c4507 100644 --- a/classes/parser.php +++ b/classes/parser.php @@ -16,10 +16,10 @@ namespace tool_dataflows; +use tool_dataflows\dataflow_lexer; use Symfony\Component\Cache\Adapter\ApcuAdapter; use Symfony\Component\Cache\Adapter\FilesystemAdapter; use Symfony\Component\ExpressionLanguage\ExpressionLanguage; -use Symfony\Component\ExpressionLanguage\Lexer; use Symfony\Component\ExpressionLanguage\Node; use Symfony\Component\ExpressionLanguage\Token; use Symfony\Component\ExpressionLanguage\TokenStream; @@ -80,9 +80,9 @@ private function __construct() { * * @return Lexer */ - private function get_lexer(): Lexer { + private function get_lexer(): dataflow_lexer { if (is_null($this->lexer)) { - $this->lexer = new Lexer(); + $this->lexer = new dataflow_lexer(); } return $this->lexer; } diff --git a/vendor/symfony/expression-language/ExpressionLanguage.php b/vendor/symfony/expression-language/ExpressionLanguage.php index 16476669..674c1dd3 100644 --- a/vendor/symfony/expression-language/ExpressionLanguage.php +++ b/vendor/symfony/expression-language/ExpressionLanguage.php @@ -15,6 +15,7 @@ use Symfony\Component\Cache\Adapter\ArrayAdapter; use Symfony\Component\ExpressionLanguage\ParserCache\ParserCacheAdapter; use Symfony\Component\ExpressionLanguage\ParserCache\ParserCacheInterface; +use tool_dataflows\dataflow_lexer; /** * Allows to compile and evaluate expressions written in your own DSL. @@ -152,7 +153,7 @@ protected function registerFunctions() private function getLexer() { if (null === $this->lexer) { - $this->lexer = new Lexer(); + $this->lexer = new dataflow_lexer(); } return $this->lexer; diff --git a/vendor/symfony/expression-language/Lexer.php b/vendor/symfony/expression-language/Lexer.php index d915b4d1..ace847b0 100644 --- a/vendor/symfony/expression-language/Lexer.php +++ b/vendor/symfony/expression-language/Lexer.php @@ -50,10 +50,6 @@ public function tokenize($expression) } $tokens[] = new Token(Token::NUMBER_TYPE, $number, $cursor + 1); $cursor += \strlen($match[0]); - } elseif (preg_match("/{'(\W[a-zA-Z_\x7f-\xff][a-zA-Z0-9_.\x7f-\xff]*)'}/A", $expression, $match, 0, $cursor)) { - // names litteral - $tokens[] = new Token(Token::NAME_TYPE, $match[1], $cursor + 1); - $cursor += \strlen($match[0]); } elseif (false !== strpos('([{', $expression[$cursor])) { // opening bracket $brackets[] = [$expression[$cursor], $cursor];