[TASK] Respect rte allow tags in translate request

With this change the RTE configuration is determined from the field to be translated.
To give deepl a list of allowed HTML tags.
To allow a correct separation in the translated text with HTML tag.

Classes/Client.php

@@ -7,6 +7,7 @@
 use Psr\Http\Message\ResponseInterface;
 use TYPO3\CMS\Core\Http\RequestFactory;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
+use WebVision\WvDeepltranslate\Domain\Dto\TranslateOptions;
 use WebVision\WvDeepltranslate\Exception\ClientNotValidUrlException;
 
 final class Client
@@ -31,21 +32,29 @@ public function __construct()
         $this->requestFactory = GeneralUtility::makeInstance(RequestFactory::class);
     }
 
-    public function translate(string $content, string $sourceLang, string $targetLang, string $glossary = ''): ResponseInterface
-    {
+    public function translate(
+        string $content,
+        string $sourceLang,
+        string $targetLang,
+        string $glossary = '',
+        ?TranslateOptions $configuration = null
+    ): ResponseInterface {
         $baseUrl = $this->buildBaseUrl('translate');
 
         $postFields = [
             'text' => $content,
             'source_lang' => $sourceLang,
             'target_lang' => $targetLang,
-            'tag_handling' => 'xml',
         ];
 
         if (!empty($glossary)) {
             $postFields['glossary_id'] = $glossary;
         }
 
+        if ($configuration instanceof TranslateOptions) {
+            $postFields = array_merge($postFields, $configuration->toArray());
+        }
+
         $postFields['formality'] = $this->configuration->getFormality();
 
         return $this->requestFactory->request($baseUrl, 'POST', $this->mergeRequiredRequestOptions([

Classes/Domain/Dto/TranslateOptions.php

@@ -0,0 +1,94 @@
+<?php
+
+declare(strict_types=1);
+
+namespace WebVision\WvDeepltranslate\Domain\Dto;
+
+final class TranslateOptions
+{
+    protected string $splitSentences = 'nonewlines';
+
+    protected bool $outlineDetection = false;
+
+    protected array $splittingTags = [];
+
+    protected array $nonSplittingTags = [];
+
+    protected array $ignoreTags = [];
+
+    protected string $tagHandling = 'xml';
+
+    public function getSplitSentences(): string
+    {
+        return $this->splitSentences;
+    }
+
+    public function setSplitSentences(string $splitSentences): void
+    {
+        $this->splitSentences = $splitSentences;
+    }
+
+    public function isOutlineDetection(): bool
+    {
+        return $this->outlineDetection;
+    }
+
+    public function setOutlineDetection(bool $outlineDetection): void
+    {
+        $this->outlineDetection = $outlineDetection;
+    }
+
+    public function getSplittingTags(): array
+    {
+        return $this->splittingTags;
+    }
+
+    public function setSplittingTags(array $splittingTags): void
+    {
+        $this->splittingTags = $splittingTags;
+    }
+
+    public function getNonSplittingTags(): array
+    {
+        return $this->nonSplittingTags;
+    }
+
+    public function setNonSplittingTags(array $nonSplittingTags): void
+    {
+        $this->nonSplittingTags = $nonSplittingTags;
+    }
+
+    public function getIgnoreTags(): array
+    {
+        return $this->ignoreTags;
+    }
+
+    public function setIgnoreTags(array $ignoreTags): void
+    {
+        $this->ignoreTags = $ignoreTags;
+    }
+
+    public function getTagHandling(): string
+    {
+        return $this->tagHandling;
+    }
+
+    public function setTagHandling(string $tagHandling): void
+    {
+        $this->tagHandling = $tagHandling;
+    }
+
+    public function toArray(): array
+    {
+        $param = [];
+        $param['tag_handling'] = $this->tagHandling;
+
+        if (!empty($this->splittingTags)) {
+            $param['outlineDetection'] = $this->outlineDetection;
+            $param['split_sentences'] = $this->splitSentences;
+            $param['splitting_tags'] = $this->splittingTags;
+        }
+
+        return $param;
+    }
+}

Classes/Hooks/TranslateHook.php

@@ -9,10 +9,12 @@
 use TYPO3\CMS\Core\Messaging\FlashMessageService;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Extbase\Object\ObjectManager;
+use WebVision\WvDeepltranslate\Domain\Dto\TranslateOptions;
 use WebVision\WvDeepltranslate\Domain\Repository\PageRepository;
 use WebVision\WvDeepltranslate\Domain\Repository\SettingsRepository;
 use WebVision\WvDeepltranslate\Exception\LanguageIsoCodeNotFoundException;
 use WebVision\WvDeepltranslate\Exception\LanguageRecordNotFoundException;
+use WebVision\WvDeepltranslate\Resolver\RichtextAllowTagsResolver;
 use WebVision\WvDeepltranslate\Service\DeeplService;
 use WebVision\WvDeepltranslate\Service\GoogleTranslateService;
 use WebVision\WvDeepltranslate\Service\LanguageService;
@@ -62,12 +64,7 @@ public function processTranslateTo_copyAction(string &$content, array $languageR
             break;
         }
 
-        if (!isset($cmdmap['localization']['custom']['srcLanguageId'])) {
-            $cmdmap['localization']['custom']['srcLanguageId'] = '';
-        }
-
         $customMode = $cmdmap['localization']['custom']['mode'] ?? null;
-        [$sourceLanguage,] = explode('-', (string)$cmdmap['localization']['custom']['srcLanguageId']);
 
         //translation mode set to deepl or google translate
         if ($customMode === null) {
@@ -79,6 +76,10 @@ public function processTranslateTo_copyAction(string &$content, array $languageR
         $translatedContent = '';
         $targetLanguageRecord = [];
 
+        $translateConfiguration = GeneralUtility::makeInstance(TranslateOptions::class);
+        $richtextAllowTagsResolver = GeneralUtility::makeInstance(RichtextAllowTagsResolver::class);
+        $translateConfiguration->setSplittingTags($richtextAllowTagsResolver->resolve($tableName, $currentRecordId));
+
         try {
             $sourceLanguageRecord = $this->languageService->getSourceLanguage(
                 $siteInformation['site']

Classes/Resolver/RichtextAllowTagsResolver.php

@@ -0,0 +1,67 @@
+<?php
+
+declare(strict_types=1);
+
+namespace WebVision\WvDeepltranslate\Resolver;
+
+use TYPO3\CMS\Backend\Utility\BackendUtility;
+use TYPO3\CMS\Core\Configuration\Richtext;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+class RichtextAllowTagsResolver
+{
+    private Richtext $richtext;
+
+    public function __construct()
+    {
+        $this->richtext = GeneralUtility::makeInstance(Richtext::class);
+    }
+
+    public function resolve(string $tableName, int $recordId): array
+    {
+        if (!isset($GLOBALS['TCA'][$tableName]['columns'])) {
+            throw new \RuntimeException('TCA Columns ist not defined', 1689950520561);
+        }
+
+        $columns = $GLOBALS['TCA'][$tableName]['columns'];
+
+        $rteTextFields = [];
+        foreach ($columns as $fieldName => $config) {
+            if (!isset($config['config']['type'])) {
+                continue;
+            }
+
+            if ($config['config']['type'] !== 'text') {
+                continue;
+            }
+
+            if (!isset($config['config']['enableRichtext'])) {
+                if ($config['config']['enableRichtext'] === false) {
+                    continue;
+                }
+            } elseif (!isset($GLOBALS['TCA'][$tableName]['types']['columnsOverrides'][$fieldName]['config']['enableRichtext'])) {
+                if ($GLOBALS['TCA'][$tableName]['types']['columnsOverrides'][$fieldName]['config']['enableRichtext'] === false) {
+                    continue;
+                }
+            }
+
+            $rteTextFields[$fieldName] = $config['config'];
+        }
+
+        if (empty($rteTextFields)) {
+            return [];
+        }
+
+        $record = BackendUtility::getRecord($tableName, $recordId);
+
+        $allowTags = [];
+        foreach ($rteTextFields as $fieldName => $config) {
+            $rteConfig = $this->richtext->getConfiguration($tableName, $fieldName, $record['pid'], $record['CType'], $config);
+            if (isset($rteConfig['processing']['allowTags'])) {
+                $allowTags = array_unique(array_merge($allowTags, $rteConfig['processing']['allowTags']), SORT_REGULAR);
+            }
+        }
+
+        return $allowTags;
+    }
+}

Tests/Functional/Resolver/Fixtures/Pages.xml

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<dataset>
+    <pages>
+        <uid>1</uid>
+        <pid>0</pid>
+        <title>DeepL-Functional-Tests</title>
+        <doktype>1</doktype>
+        <is_siteroot>1</is_siteroot>
+    </pages>
+    <pages>
+        <uid>3</uid>
+        <pid>0</pid>
+        <title>DeepL-Functional-Tests BS</title>
+        <doktype>1</doktype>
+        <is_siteroot>1</is_siteroot>
+    </pages>
+    <tt_content>
+        <uid>1</uid>
+        <pid>1</pid>
+        <header>DeepL-Functional-Test Element</header>
+        <CType>text</CType>
+        <bodytext></bodytext>
+    </tt_content>
+</dataset>

Tests/Functional/Resolver/RichtextAllowTagsResolverTest.php

@@ -0,0 +1,50 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Functional\Resolver;
+
+use Nimut\TestingFramework\TestCase\FunctionalTestCase;
+use TYPO3\CMS\Core\Configuration\Loader\YamlFileLoader;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use WebVision\WvDeepltranslate\Resolver\RichtextAllowTagsResolver;
+
+class RichtextAllowTagsResolverTest extends FunctionalTestCase
+{
+    /**
+     * @var string[]
+     */
+    protected $testExtensionsToLoad = [
+        'typo3conf/ext/wv_deepltranslate',
+    ];
+
+    /**
+     * @var string[]
+     */
+    protected $coreExtensionsToLoad = [
+        'rte_ckeditor',
+    ];
+
+    protected function setUp(): void
+    {
+        parent::setUp();
+
+        $this->importDataSet(__DIR__ . '/Fixtures/Pages.xml');
+    }
+
+    /**
+     * @test
+     */
+    public function findRteConfigurationAllowTagsByRecords(): void
+    {
+        $richtextAllowTagsResolver = GeneralUtility::makeInstance(RichtextAllowTagsResolver::class);
+        $allowTags = $richtextAllowTagsResolver->resolve('tt_content', 1);
+
+        static::assertTrue((bool)array_search('em', $allowTags));
+
+        $yamlLoader = GeneralUtility::makeInstance(YamlFileLoader::class);
+        $config = $yamlLoader->load('EXT:rte_ckeditor/Configuration/RTE/Processing.yaml');
+
+        static::assertSame($config['processing']['allowTags'], $allowTags);
+    }
+}

Tests/Functional/Services/DeeplServiceTest.php

@@ -53,6 +53,29 @@ public function translateContentFromDeToEn(): void
         static::assertSame('I would like to be translated!', $responseObject['translations'][0]['text']);
     }
 
+    /**
+     * @test
+     */
+    public function translateContentAndRespectHtmlTags(): void
+    {
+        if (defined('DEEPL_MOCKSERVER_USED') && DEEPL_MOCKSERVER_USED === true) {
+            static::markTestSkipped(__METHOD__ . ' skipped, because DEEPL MOCKSERVER do not support EN as TARGET language.');
+        }
+
+        $deeplService = GeneralUtility::makeInstance(DeeplService::class);
+
+        $responseObject = $deeplService->translateRequest(
+            '<p>Important species in blueberry include the Western flower thrips (<em>Frankliniella occidentalis</em>) and Chilli thrips (<em>Scirtothrips dorsalis</em>).</p>',
+            'DE',
+            'EN'
+        );
+
+        static::assertSame(
+            '<p>Zu den für Heidelbeeren wichtigen Arten gehören der Westliche Blütenthrips (<em>Frankliniella occidentalis</em>) und der Chili-Thrips (<em>Scirtothrips dorsalis</em>).</p>',
+            $responseObject['translations'][0]['text']
+        );
+    }
+
     /**
      * @test
      */

composer.json

@@ -91,6 +91,7 @@
 		"typo3/cms-frontend": "^9.5 || ^10.4 || ^11.5",
 		"typo3/cms-info": "^9.5 || ^10.4 || ^11.5",
 		"typo3/cms-lowlevel": "^9.5 || ^10.4 || ^11.5",
+		"typo3/cms-rte-ckeditor": "^9.5 || ^10.4 || ^11.5",
 		"typo3/cms-tstemplate": "^9.5 || ^10.4 || ^11.5",
 		"typo3/cms-workspaces": "^9.5 || ^10.4 || ^11.5",
 		"typo3/coding-standards": "^0.5"