issue 907 Add option for regex replace and null value on input flow

classes/local/step/flow_transformer_regex.php

@@ -43,6 +43,7 @@ public static function form_define_fields(): array {
         return [
             'pattern' => ['type' => PARAM_RAW, 'required' => true],
             'field' => ['type' => PARAM_TEXT, 'required' => true],
+            'replacenull' => ['type' => PARAM_BOOL, 'required' => true],
         ];
     }
 
@@ -82,6 +83,17 @@ public function form_add_custom_inputs(\MoodleQuickForm &$mform) {
             '',
             get_string('flow_transformer_regex:field_help', 'tool_dataflows')
         );
+        $mform->addElement(
+            'checkbox',
+            'config_replacenull',
+            get_string('flow_transformer_regex:replacenull', 'tool_dataflows')
+        );
+        $mform->addElement(
+            'static',
+            'config_replacenull_help',
+            '',
+            get_string('flow_transformer_regex:replacenull_help', 'tool_dataflows')
+        );
     }
 
     /**
@@ -95,7 +107,90 @@ public function execute($input = null) {
         $pattern = $this->stepdef->config->pattern;
         $field = $this->stepdef->config->field;
         $haystack = $variables->evaluate($field);
+        $hasnamedcapturegroups = false;
+
+        // Get options from pattern. SED style regex to seperate match and replace parameters.
+        // TODO: add support for flags eg. /g /m (global, multi-line etc).
+        [$replace, $pattern, $flags] = self::get_pattern_options($pattern);
+
+        // Process either match or replace.
+        if ($replace) {
+            $result = self::regex_replace($pattern, $haystack);
+        } else {
+            [$hasnamedcapturegroups, $result] = self::regex_match($pattern, $haystack);
+        }
+
+        // Support named capture groups.
+        // Otherwise set the first match (or null) to the field named as the step alias.
+        if ($hasnamedcapturegroups) {
+            $input = (object) array_merge(
+                (array) $input, $result);
+        } else {
+            $uniquekey = $variables->get('alias');
+            $input->$uniquekey = $result[0];
+        }
+
+        return $input;
+    }
+
+    /**
+     * Gets match and replace parameters from SED style regex.
+     * This determines if preg_match or preg_replace is used.
+     * @param string $fullpattern SED style regex
+     * @return array match and replace parameters
+     */
+    private function get_pattern_options(string $fullpattern): array {
         $matches = [];
+        $pattern = '/(.*?)(\/.*\/)(.*)/';
+        preg_match($pattern, $fullpattern, $matches);
+        // Having a 's/' at the start of the pattern denotes search and replace.
+        // eg 's/<search regex>/<replacement>/<flags>'.
+        $replace = $matches[1] === 's';
+        $pattern = $matches[2];
+        $flags = $matches[3];
+
+        return [$replace, $pattern, $flags];
+    }
+
+    /**
+     * Gets the search and replace parameters from the fullpattern.
+     * @param string $fullpattern SED style regex
+     * @return array search and replace parameters
+     */
+    private function get_pattern_substitution(string $fullpattern): array {
+        $matches = [];
+        $pattern = '/(.*[^\\\\]\/)(.*)\//';
+        preg_match($pattern, $fullpattern, $matches);
+        $pattern = $matches[1];
+        $substitution = $matches[2];
+
+        return [$pattern, $substitution];
+    }
+
+    /**
+     * Process regex search and replace and returns result.
+     * @param string $pattern
+     * @param string $haystack
+     * @return array Results of the regex search and replace
+     */
+    private function regex_replace(string $pattern, string $haystack): array {
+        $result = [];
+        [$pattern, $substitution] = self::get_pattern_substitution($pattern);
+
+        $result[] = preg_replace($pattern, $substitution, $haystack);
+        return $result;
+    }
+
+    /**
+     * Process regex matches and returns results.
+     * @param string $pattern
+     * @param string $haystack
+     * @return array Results of the regex match
+     */
+    private function regex_match(string $pattern, string $haystack): array {
+        $matches = [];
+        $result = [];
+
         preg_match($pattern, $haystack, $matches);
 
         // Support named capture groups.
@@ -106,16 +201,15 @@ public function execute($input = null) {
             }
 
             $hasnamedcapturegroups = true;
-            $input->$key = $value;
+            $result[$key] = $value;
         }
 
         // Otherwise set the first match (or null) to the field named as the step alias.
         if (!$hasnamedcapturegroups) {
             // Capture the first matched string as a variable.
-            $uniquekey = $variables->get('alias');
-            $input->$uniquekey = $matches[0] ?? null;
+            $result[] = $matches[0] ?? null;
         }
 
-        return $input;
+        return [$hasnamedcapturegroups, $result];
     }
 }

classes/local/variables/var_step.php

@@ -62,4 +62,27 @@ public function init() {
             $this->set("states.$state", null);
         }
     }
+
+    /**
+     * Magic getter - which allows the user to get values directly instead of via ->get('name')
+     *
+     * @param      string $name of the property to get
+     * @return     mixed
+     */
+    public function __get($name) {
+        $methodname = 'get_' . $name;
+        if (method_exists($this, $methodname)) {
+            return $this->$methodname();
+        }
+        return $this->get($name);
+    }
+
+    /**
+     * Gets the stepdef.
+     *
+     * @return step
+     */
+    private function get_stepdef(): step {
+        return $this->stepdef;
+    }
 }

classes/local/variables/var_value.php

@@ -192,6 +192,20 @@ private function make_reference_tree(): \stdClass {
         foreach ($this->references as $name => $obj) {
             $value = $obj->get_resolved(false);
 
+            // This function is called during initialisation and when processing steps.
+            // Check if a step is being processed with specific null handling.
+            if (is_null($value)) {
+                if (get_class($this->parent) === var_step::class) {
+                    $step = $this->parent;
+
+                    // Currently only option is to replace the null with an empty string.
+                    $replacenull = $step->stepdef->config->replacenull;
+                    if ($replacenull) {
+                        $value = '';
+                    }
+                }
+            }
+
             // Do not add the value if it is not set. This should result in the expression remaining unresolved in the evalutaion.
             if (is_null($value)) {
                 continue;

lang/en/tool_dataflows.php

@@ -524,6 +524,8 @@
 $string['flow_transformer_regex:pattern_help'] = "Checked against the input field. The first match will be set on the record using this step's alias as the key. Named capture groups are used instead, if set.";
 $string['flow_transformer_regex:field'] = 'Input field';
 $string['flow_transformer_regex:field_help'] = "This field must be a string that will be processed using the regex pattern";
+$string['flow_transformer_regex:replacenull'] = 'Replace null with empty string';
+$string['flow_transformer_regex:replacenull_help'] = "Replaces input values that are null with empty string. Otherwise null values are ignored.";
 
 // Wait connector.
 $string['connector_wait:timesec'] = 'Time in seconds';

tests/tool_dataflows_flow_transformer_regex_test.php

@@ -86,11 +86,17 @@ protected function tearDown(): void {
      *
      * @dataProvider regex_provider
      * @covers \tool_dataflows\local\step\flow_transformer_regex
-     * @param array $data
+     * @param array|string $data
      * @param array $config
-     * @param array $expected
+     * @param array|string $expected
      */
-    public function test_regex(array $data, array $config, array $expected) {
+    public function test_regex($data, array $config, $expected) {
+        // Add default 'field' value.
+        $config['field'] = '${{steps.reader.record.test}}';
+
+        // Convert single test data to array format.
+        $expected = is_array($expected) ? $expected : [['test' => $data, 'regex' => $expected]];
+        $data = is_array($data) ? $data : [['test' => $data]];
 
         // Perform the test.
         set_config('permitted_dirs', $this->basedir, 'tool_dataflows');
@@ -111,17 +117,29 @@ public function test_regex(array $data, array $config, array $expected) {
      * @return array[]
      */
     public static function regex_provider(): array {
-        return [
-            [self::TEST_DATA, self::CONFIG, self::EXPECTED],
-            [
-                [['test' => '1,2,3,4,5,6,7,8,9']],
-                [
-                    'field' => '${{steps.reader.record.test}}',
-                    'pattern' => '/4\,5\,6/',
-                ],
-                [['test' => '1,2,3,4,5,6,7,8,9', 'regex' => '4,5,6']],
-            ],
-        ];
+        $testcases = [];
+
+        // Add testcases here.
+        // format:
+        // ['test' => <test string>, ['pattern' => <regex>, (optional) 'replacenull' => <0 or 1>], 'expected' => <expected value>].
+
+        $testcases[] = [ self::TEST_DATA, self::CONFIG, self::EXPECTED ];
+
+        // Test cases for 'match'.
+        $testcases[] = ['test' => '1,2,3,4,5,6,7,8,9', ['pattern' => '/4\,5\,6/'], 'expected' => '4,5,6'];
+
+        // Test cases for 'replace'.
+        $testcases[] = ['test' => 'hello earth', ['pattern' => 's/earth/world/g'], 'expected' => 'hello world'];
+        $testcases[] = ['test' => 'hello earth world', ['pattern' => 's/\searth//g'], 'expected' => 'hello world'];
+        $testcases[] = ['test' => 'ab123fg', ['pattern' => 's/123/cde/'], 'expected' => 'abcdefg'];
+
+        // Test case for null value.
+        // Without empty string substitution (replacenull) the 'field' value is returned.
+        $testcases[] = ['test' => null, ['pattern' => 's/any/pattern/', 'replacenull' => 0],
+            'expected' => '${{steps.reader.record.test}}'];
+        $testcases[] = ['test' => null, ['pattern' => 's/any/pattern/', 'replacenull' => 1], 'expected' => ''];
+
+        return $testcases;
     }
 
     /**

version.php

@@ -25,7 +25,7 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version = 2024101000;
+$plugin->version = 2024102500;
 $plugin->release = 2024101000;
 $plugin->requires = 2022112800;    // Our lowest supported Moodle (3.3.0).
 $plugin->supported = [400, 402];