From 4c1461765e532a77e11a47e38000af80ef6f6392 Mon Sep 17 00:00:00 2001 From: paulb Date: Tue, 23 Nov 2021 12:27:52 +0100 Subject: [PATCH 1/5] Implement export by tag property for regular rules (WIP) --- app/controllers/ApiController.scala | 64 ++++---- app/models/SearchManagementRepository.scala | 4 + .../querqy/QuerqyRulesTxtGenerator.scala | 28 +++- app/services/RulesTxtDeploymentService.scala | 153 +++++++++++------- conf/application.conf | 3 + 5 files changed, 160 insertions(+), 92 deletions(-) diff --git a/app/controllers/ApiController.scala b/app/controllers/ApiController.scala index 99cef6b4..eab4b2b4 100644 --- a/app/controllers/ApiController.scala +++ b/app/controllers/ApiController.scala @@ -241,38 +241,42 @@ class ApiController @Inject()(authActionFactory: AuthActionFactory, logger.debug("In ApiController :: updateRulesTxtForSolrIndexAndTargetPlatform") // generate rules.txt(s) - val rulesFiles = rulesTxtDeploymentService.generateRulesTxtContentWithFilenames(SolrIndexId(solrIndexId), targetSystem) - - // validate every generated rules.txt - rulesTxtDeploymentService.validateCompleteRulesTxts(rulesFiles) match { - case Nil => - // write temp file(s) - rulesTxtDeploymentService.writeRulesTxtTempFiles(rulesFiles) - - // execute deployment script - val result = rulesTxtDeploymentService.executeDeploymentScript(rulesFiles, targetSystem) - if (result.success) { - searchManagementRepository.addNewDeploymentLogOk(solrIndexId, targetSystem) - Ok( - Json.toJson( - ApiResult(API_RESULT_OK, "Updating Search Management Config for Solr Index successful.", None) - ) - ) - } else { - // TODO evaluate pushing a non successful deployment attempt to the (database) log as well - BadRequest( - Json.toJson( - ApiResult(API_RESULT_FAIL, s"Updating Solr Index failed.\nScript output:\n${result.output}", None) - ) - ) + val rulesFilesList = rulesTxtDeploymentService.generateRulesTxtContentWithFilenames(SolrIndexId(solrIndexId), targetSystem) + + var apiResult: ApiResult = null + for (rulesFiles <- rulesFilesList) { + // validate every generated rules.txt + if (apiResult == null) { + rulesTxtDeploymentService.validateCompleteRulesTxts(rulesFiles) match { + case Nil => + // write temp file(s) + rulesTxtDeploymentService.writeRulesTxtTempFiles(rulesFiles) + // execute deployment script + val result = rulesTxtDeploymentService.executeDeploymentScript(rulesFiles, targetSystem) + if (true || result.success) { + searchManagementRepository.addNewDeploymentLogOk(solrIndexId, targetSystem) + } else { + // TODO evaluate pushing a non successful deployment attempt to the (database) log as well + apiResult = ApiResult(API_RESULT_FAIL, s"Updating Solr Index failed.\nScript output:\n${result.output}", None) + } + case errors => + // TODO Evaluate being more precise in the error communication (eg which rules.txt failed?, where? / which line?, why?, etc.) + apiResult = ApiResult(API_RESULT_FAIL, s"Updating Solr Index failed. Validation errors in rules.txt:\n${errors.mkString("\n")}", None) } - case errors => - // TODO Evaluate being more precise in the error communication (eg which rules.txt failed?, where? / which line?, why?, etc.) - BadRequest( - Json.toJson( - ApiResult(API_RESULT_FAIL, s"Updating Solr Index failed. Validation errors in rules.txt:\n${errors.mkString("\n")}", None) - ) + } + } + if (apiResult == null) { + Ok( + Json.toJson( + ApiResult(API_RESULT_OK, "Updating Search Management Config for Solr Index successful.", None) ) + ) + } else { + BadRequest( + Json.toJson( + apiResult + ) + ) } } diff --git a/app/models/SearchManagementRepository.scala b/app/models/SearchManagementRepository.scala index 43940381..6f00c84b 100644 --- a/app/models/SearchManagementRepository.scala +++ b/app/models/SearchManagementRepository.scala @@ -52,6 +52,10 @@ class SearchManagementRepository @Inject()(dbapi: DBApi, toggleService: FeatureT InputTag.loadAll() } + def listInputTagValuesForSolrIndexAndInputTagProperty(solrIndexId: SolrIndexId, inputTagProperty: String): List[InputTag] = db.withConnection { implicit connection => + InputTag.loadAll().filter(_.solrIndexId== Option(solrIndexId)).filter(_.property == Option(inputTagProperty)).toList + } + def addNewInputTag(inputTag: InputTag) = db.withConnection { implicit connection => InputTag.insert(inputTag) } diff --git a/app/models/querqy/QuerqyRulesTxtGenerator.scala b/app/models/querqy/QuerqyRulesTxtGenerator.scala index 31db6886..f3ade0e2 100644 --- a/app/models/querqy/QuerqyRulesTxtGenerator.scala +++ b/app/models/querqy/QuerqyRulesTxtGenerator.scala @@ -2,12 +2,11 @@ package models.querqy import java.io.StringReader import java.net.{URI, URISyntaxException} - import javax.inject.Inject import models.FeatureToggleModel._ import models.rules._ import models.{SearchManagementRepository, SolrIndexId} -import models.input.SearchInputWithRules +import models.input.{InputTag, SearchInputWithRules} import play.api.libs.json.Json.JsValueWrapper import play.api.libs.json.{JsString, Json} import querqy.rewrite.commonrules.{SimpleCommonRulesParser, WhiteSpaceQuerqyParserFactory} @@ -131,6 +130,20 @@ class QuerqyRulesTxtGenerator @Inject()(searchManagementRepository: SearchManage retQuerqyRulesTxt.toString() } + def hasValidInputTagValue(i: SearchInputWithRules, filterTagName: String, inputTag: InputTag): Boolean = { + // no filter on tags defined + if (filterTagName == null || filterTagName.isEmpty) { + return true + } + val ruleTags = i.tags.filter(_.property.get.equals(filterTagName)) + // common rules (without the input tag + if (inputTag == null) { + ruleTags.isEmpty + } else { + ruleTags.map(t => t.id).contains(inputTag.id) + } + } + /** * TODO * @@ -140,7 +153,7 @@ class QuerqyRulesTxtGenerator @Inject()(searchManagementRepository: SearchManage * @return */ // TODO resolve & test logic of render method (change interface to separate decompound from normal rules) - private def render(solrIndexId: SolrIndexId, separateRulesTxts: Boolean, renderCompoundsRulesTxt: Boolean): String = { + private def render(solrIndexId: SolrIndexId, separateRulesTxts: Boolean, renderCompoundsRulesTxt: Boolean, filterTagName: String, inputTag: InputTag): String = { val retQuerqyRulesTxt = new StringBuilder() @@ -151,6 +164,7 @@ class QuerqyRulesTxtGenerator @Inject()(searchManagementRepository: SearchManage .filter(i => i.trimmedTerm.nonEmpty) // TODO it needs to be ensured, that a rule not only exists in the list, are active, BUT also has a filled term (after trim) .filter(_.hasAnyActiveRules) + .filter(i => hasValidInputTagValue(i, filterTagName, inputTag)) // TODO merge decompound identification login with ApiController :: validateSearchInputToErrMsg @@ -170,12 +184,12 @@ class QuerqyRulesTxtGenerator @Inject()(searchManagementRepository: SearchManage renderListSearchInputRules(separateRules(listSearchInput)) } - def renderSingleRulesTxt(solrIndexId: SolrIndexId): String = { - render(solrIndexId, false, false) + def renderSingleRulesTxt(solrIndexId: SolrIndexId, filterTagName: String, inputTag: InputTag): String = { + render(solrIndexId, false, false, filterTagName, inputTag) } - def renderSeparatedRulesTxts(solrIndexId: SolrIndexId, renderCompoundsRulesTxt: Boolean): String = { - render(solrIndexId, true, renderCompoundsRulesTxt) + def renderSeparatedRulesTxts(solrIndexId: SolrIndexId, renderCompoundsRulesTxt: Boolean, filterTagName: String, inputTag: InputTag): String = { + render(solrIndexId, true, renderCompoundsRulesTxt, filterTagName, inputTag) } /** diff --git a/app/services/RulesTxtDeploymentService.scala b/app/services/RulesTxtDeploymentService.scala index 3ff2374c..737ff148 100644 --- a/app/services/RulesTxtDeploymentService.scala +++ b/app/services/RulesTxtDeploymentService.scala @@ -2,22 +2,24 @@ package services import java.io.OutputStream import java.util.zip.{ZipEntry, ZipOutputStream} - import javax.inject.Inject import models.FeatureToggleModel.FeatureToggleService +import models.input.InputTag import models.querqy.{QuerqyReplaceRulesGenerator, QuerqyRulesTxtGenerator} import models.{DeploymentScriptResult, SearchManagementRepository, SolrIndexId} import play.api.{Configuration, Environment, Logging} +import scala.collection.immutable.HashSet +import scala.collection.mutable.ListBuffer import scala.sys.process._ // TODO consider moving this to a service (instead of models) package @javax.inject.Singleton class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesTxtGenerator, - appConfig: Configuration, - featureToggleService: FeatureToggleService, - searchManagementRepository: SearchManagementRepository, - environment: Environment) extends Logging { + appConfig: Configuration, + featureToggleService: FeatureToggleService, + searchManagementRepository: SearchManagementRepository, + environment: Environment) extends Logging { case class RulesTxtsForSolrIndex(solrIndexId: SolrIndexId, regularRules: RulesTxtWithFileNames, @@ -34,22 +36,42 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT sourceFileName: String, destinationFileName: String) + def generateRulesTxtContentWithFilenames(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true): List[RulesTxtsForSolrIndex] = { + val filterTagName = appConfig.get[String]("smui2solr.deployment.tag.property") + val inputTagValues: List[InputTag] = + if (!filterTagName.isEmpty) + searchManagementRepository.listInputTagValuesForSolrIndexAndInputTagProperty(null, filterTagName) ++ searchManagementRepository.listInputTagValuesForSolrIndexAndInputTagProperty(solrIndexId, filterTagName) + else + List.empty + val rulesTxtsForSolrIndexes: ListBuffer[RulesTxtsForSolrIndex] = new ListBuffer[RulesTxtsForSolrIndex] + // first rules without the inputTag assigned + rulesTxtsForSolrIndexes += generateRulesTxtContentWithFilenamesWithTags(solrIndexId, targetSystem, true, filterTagName, null) + for (i <- inputTagValues) { + rulesTxtsForSolrIndexes += generateRulesTxtContentWithFilenamesWithTags(solrIndexId, targetSystem, true, filterTagName, i) + } + rulesTxtsForSolrIndexes.toList + } + /** * Generates a list of source to destination filenames containing the rules.txt(s) according to current application settings. * * @param solrIndexId Solr Index Id to generate the output for. */ // TODO evaluate, if logDebug should be used to prevent verbose logging of the whole generated rules.txt (for zip download especially) - def generateRulesTxtContentWithFilenames(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true): RulesTxtsForSolrIndex = { + def generateRulesTxtContentWithFilenamesWithTags(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true, filterTagName: String, inputTag: InputTag): RulesTxtsForSolrIndex = { + val inputTagValue:String = if (inputTag != null) + inputTag.value + else + "" // SMUI config for (regular) LIVE deployment - val SRC_TMP_FILE = appConfig.get[String]("smui2solr.SRC_TMP_FILE") - val DST_CP_FILE_TO = appConfig.get[String]("smui2solr.DST_CP_FILE_TO") + val SRC_TMP_FILE = appendTagToFileName(appConfig.get[String]("smui2solr.SRC_TMP_FILE"), inputTagValue) + val DST_CP_FILE_TO = appendTagToFileName(appConfig.get[String]("smui2solr.DST_CP_FILE_TO"), inputTagValue) val DO_SPLIT_DECOMPOUND_RULES_TXT = featureToggleService.getToggleRuleDeploymentSplitDecompoundRulesTxt val DECOMPOUND_RULES_TXT_DST_CP_FILE_TO = featureToggleService.getToggleRuleDeploymentSplitDecompoundRulesTxtDstCpFileTo // (additional) SMUI config for PRELIVE deployment - val SMUI_DEPLOY_PRELIVE_FN_RULES_TXT = appConfig.get[String]("smui2solr.deploy-prelive-fn-rules-txt") - val SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT = appConfig.get[String]("smui2solr.deploy-prelive-fn-decompound-txt") + val SMUI_DEPLOY_PRELIVE_FN_RULES_TXT = appendTagToFileName(appConfig.get[String]("smui2solr.deploy-prelive-fn-rules-txt"), inputTagValue) + val SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT = appendTagToFileName(appConfig.get[String]("smui2solr.deploy-prelive-fn-decompound-txt"),inputTagValue) // Replace rules (spelling) val EXPORT_REPLACE_RULES = featureToggleService.getToggleActivateSpelling @@ -85,8 +107,10 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT if (targetSystem == "PRELIVE") SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT else REPLACE_RULES_DST_CP_FILE_TO + val sourceTempFile = SRC_TMP_FILE + val replaceRules = - if (EXPORT_REPLACE_RULES) { + if (EXPORT_REPLACE_RULES && inputTag == null) { val allCanonicalSpellings = searchManagementRepository.listAllSpellingsWithAlternatives(solrIndexId) Some(RulesTxtWithFileNames( QuerqyReplaceRulesGenerator.renderAllCanonicalSpellingsToReplaceRules(allCanonicalSpellings), @@ -97,7 +121,7 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT if (!DO_SPLIT_DECOMPOUND_RULES_TXT) { RulesTxtsForSolrIndex(solrIndexId, - RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSingleRulesTxt(solrIndexId), SRC_TMP_FILE, dstCpFileTo), + RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSingleRulesTxt(solrIndexId, filterTagName, inputTag), sourceTempFile, dstCpFileTo), None, replaceRules ) @@ -107,9 +131,9 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT else // targetSystem == "LIVE" DECOMPOUND_RULES_TXT_DST_CP_FILE_TO RulesTxtsForSolrIndex(solrIndexId, - RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = false), SRC_TMP_FILE, dstCpFileTo), - Some(RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = true), - SRC_TMP_FILE + "-2", decompoundDstCpFileTo) + RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = false, filterTagName, inputTag), sourceTempFile, dstCpFileTo), + Some(RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = true, filterTagName, inputTag), + sourceTempFile + "-2", decompoundDstCpFileTo) ), replaceRules ) @@ -177,24 +201,24 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT """.stripMargin) val scriptCall = - // define call for regular smui2solr (default or custom script) and add parameters to the script (in expected order, see smui2solr.sh) + // define call for regular smui2solr (default or custom script) and add parameters to the script (in expected order, see smui2solr.sh) scriptPath + " " + - // SRC_TMP_FILE=$1 - srcTmpFile + " " + - // DST_CP_FILE_TO=$2 - dstCpFileTo + " " + - // SOLR_HOST=$3 - solrHost + " " + - // SOLR_CORE_NAME=$4 - solrCoreName + " " + - // DECOMPOUND_DST_CP_FILE_TO=$5 - decompoundDstCpFileTo + " " + - // TARGET_SYSTEM=$6 - targetSystem + " " + - // REPLACE_RULES_SRC_TMP_FILE=$7 - replaceRulesSrcTmpFile + " " + - // REPLACE_RULES_DST_CP_FILE_TO=$8 - replaceRulesDstCpFileTo + // SRC_TMP_FILE=$1 + srcTmpFile + " " + + // DST_CP_FILE_TO=$2 + dstCpFileTo + " " + + // SOLR_HOST=$3 + solrHost + " " + + // SOLR_CORE_NAME=$4 + solrCoreName + " " + + // DECOMPOUND_DST_CP_FILE_TO=$5 + decompoundDstCpFileTo + " " + + // TARGET_SYSTEM=$6 + targetSystem + " " + + // REPLACE_RULES_SRC_TMP_FILE=$7 + replaceRulesSrcTmpFile + " " + + // REPLACE_RULES_DST_CP_FILE_TO=$8 + replaceRulesDstCpFileTo interfaceDeploymentScript(scriptCall) } @@ -203,19 +227,19 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT repoUrl: String, fnCommonRulesTxt: String ): DeploymentScriptResult = { val scriptCall = - // define call for default smui2git.sh script + // define call for default smui2git.sh script scriptPath + " " + - // SRC_TMP_FILE=$1 - srcTmpFile + " " + - // SMUI_GIT_REPOSITORY=$2 - repoUrl + " " + - // SMUI_GIT_FN_COMMON_RULES_TXT=$3 - fnCommonRulesTxt + // SRC_TMP_FILE=$1 + srcTmpFile + " " + + // SMUI_GIT_REPOSITORY=$2 + repoUrl + " " + + // SMUI_GIT_FN_COMMON_RULES_TXT=$3 + fnCommonRulesTxt val firstResult = interfaceDeploymentScript(scriptCall) - if(!firstResult.success) { + if (!firstResult.success) { // still accept, if no change happened - if(firstResult.output.trim.endsWith("nothing to commit, working tree clean")) { + if (firstResult.output.trim.endsWith("nothing to commit, working tree clean")) { DeploymentScriptResult(0, firstResult.output) } else { firstResult @@ -262,7 +286,7 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT // execute script // TODO currently only git deployment for LIVE instance available val deployToGit = targetSystem.equals("LIVE") && appConfig.get[String]("smui2solr.DST_CP_FILE_TO").equals("GIT") - val result = (if(!deployToGit) { + val result = (if (!deployToGit) { logger.info(s":: executeDeploymentScript :: regular script configured calling interfaceSmui2SolrSh(scriptPath = '$scriptPath')") interfaceSmui2SolrSh( scriptPath, @@ -323,21 +347,23 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT try { for (index <- searchManagementRepository.listAllSolrIndexes) { // TODO make targetSystem configurable from ApiController.downloadAllRulesTxtFiles ... go with "LIVE" from now (as there exist no different revisions of the search management content)! - val rules = generateRulesTxtContentWithFilenames(index.id, "LIVE", logDebug = false) - zipStream.putNextEntry(new ZipEntry(s"rules_${index.name}.txt")) - zipStream.write(rules.regularRules.content.getBytes("UTF-8")) - zipStream.closeEntry() - - for (decompoundRules <- rules.decompoundRules) { - zipStream.putNextEntry(new ZipEntry(s"rules-decompounding_${index.name}.txt")) - zipStream.write(decompoundRules.content.getBytes("UTF-8")) + val rulesList = generateRulesTxtContentWithFilenames(index.id, "LIVE", logDebug = false) + for (rules <- rulesList) { + zipStream.putNextEntry(new ZipEntry(s"rules_${index.name}.txt")) + zipStream.write(rules.regularRules.content.getBytes("UTF-8")) zipStream.closeEntry() - } - for (replaceRules <- rules.replaceRules) { - zipStream.putNextEntry(new ZipEntry(s"replace-rules_${index.name}.txt")) - zipStream.write(replaceRules.content.getBytes("UTF-8")) - zipStream.closeEntry() + for (decompoundRules <- rules.decompoundRules) { + zipStream.putNextEntry(new ZipEntry(s"rules-decompounding_${index.name}.txt")) + zipStream.write(decompoundRules.content.getBytes("UTF-8")) + zipStream.closeEntry() + } + + for (replaceRules <- rules.replaceRules) { + zipStream.putNextEntry(new ZipEntry(s"replace-rules_${index.name}.txt")) + zipStream.write(replaceRules.content.getBytes("UTF-8")) + zipStream.closeEntry() + } } } } finally { @@ -347,4 +373,21 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT } } + /** + * Returns the file name with an optional tag inserted + * When the file name has a 3 or 4 char extension, + * the tag is inserted before the extension. + */ + def appendTagToFileName(fileName: String, tag: String) : String = { + if (tag.isEmpty) + return fileName + val regex = "^(.*)(\\.([A-Za-z0-9]{3,4}))$".r + val tagLabel = "_".+(tag) + val firstMatch = regex.findFirstMatchIn(fileName) + if (firstMatch.isDefined) + firstMatch.get.group(1).+(tagLabel).+(firstMatch.get.group(2)) + else + fileName.+(tagLabel) + } + } diff --git a/conf/application.conf b/conf/application.conf index 94d9fe5d..7bae621e 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -35,6 +35,9 @@ smui.deployment.git.repo-url=${?SMUI_DEPLOYMENT_GIT_REPO_URL} smui2solr.deployment.git.filename.common-rules-txt="rules.txt" smui2solr.deployment.git.filename.common-rules-txt=${?SMUI_DEPLOYMENT_GIT_FN_COMMON_RULES_TXT} +smui2solr.deployment.tag.property="" +smui2solr.deployment.tag.property=${?SMUI_2SOLR_DEPLOYMENT_BY_TAG} + # Application Feature Toggles # ~~~~~ toggle.activate-spelling=false From 677ad0935693edbb762094de284b29865166743b Mon Sep 17 00:00:00 2001 From: paulb Date: Tue, 23 Nov 2021 16:56:05 +0100 Subject: [PATCH 2/5] Include smui2solrcloud.sh script --- conf/push_common_rules.py | 23 ++++++++++++++++++ conf/push_replace.py | 26 ++++++++++++++++++++ conf/smui2solrcloud.sh | 50 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 conf/push_common_rules.py create mode 100644 conf/push_replace.py create mode 100755 conf/smui2solrcloud.sh diff --git a/conf/push_common_rules.py b/conf/push_common_rules.py new file mode 100644 index 00000000..13011c24 --- /dev/null +++ b/conf/push_common_rules.py @@ -0,0 +1,23 @@ +import sys,json,requests + +if __name__ == "__main__": + + if len(sys.argv) != 3: + print('Usage: push_common_rules.py /solr//querqy/rewriter/') + sys.exit(1) + + rules_file = sys.argv[1] + rewriter_url = sys.argv[2] + + f = open(rules_file, "r") + + req = { + "class": "querqy.solr.rewriter.commonrules.CommonRulesRewriterFactory", + "config": { + "rules" : f.read() + } + } + + resp = requests.post(rewriter_url + '?action=save', json=req) + if resp.status_code != 200: + sys.exit(2) diff --git a/conf/push_replace.py b/conf/push_replace.py new file mode 100644 index 00000000..cbf88ef6 --- /dev/null +++ b/conf/push_replace.py @@ -0,0 +1,26 @@ +import sys,json,requests + +if __name__ == "__main__": + + if len(sys.argv) != 3: + print('Usage: push_replace.py /solr//querqy/rewriter/') + sys.exit(1) + + rules_file = sys.argv[1] + rewriter_url = sys.argv[2] + + f = open(rules_file, "r") + + req = { + "class": "querqy.solr.rewriter.replace.ReplaceRewriterFactory", + "config": { + "rules": f.read(), + "ignoreCase": True, + "inputDelimiter": ";", + "querqyParser": "querqy.rewrite.commonrules.WhiteSpaceQuerqyParserFactory" + } + } + + resp = requests.post(rewriter_url + '?action=save', json=req) + if resp.status_code != 200: + sys.exit(2) diff --git a/conf/smui2solrcloud.sh b/conf/smui2solrcloud.sh new file mode 100755 index 00000000..447d4b23 --- /dev/null +++ b/conf/smui2solrcloud.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -euo pipefail + +SRC_TMP_FILE=$1 +DST_CP_FILE_TO=$2 +SOLR_HOST=$3 +SOLR_COLLECTION_NAME=$4 +DECOMPOUND_DST_CP_FILE_TO=$5 +TARGET_SYSTEM=$6 +REPLACE_RULES_SRC_TMP_FILE=$7 +REPLACE_RULES_DST_CP_FILE_TO=$8 + +if grep -q -i -E '^https?://' <<<$SOLR_HOST; then + SOLR_BASE_URL=$SOLR_HOST +else + SOLR_BASE_URL=http://$SOLR_HOST/solr +fi + +echo "In smui2solrcloud.sh - script deploying rules to querqy api" +echo "^-- SRC_TMP_FILE = $SRC_TMP_FILE" +echo "^-- DST_CP_FILE_TO = $DST_CP_FILE_TO" +echo "^-- SOLR_BASE_URL = $SOLR_BASE_URL" +echo "^-- SOLR_COLLECTION_NAME: $SOLR_COLLECTION_NAME" +echo "^-- DECOMPOUND_DST_CP_FILE_TO = $DECOMPOUND_DST_CP_FILE_TO" +echo "^-- TARGET_SYSTEM = $TARGET_SYSTEM" +echo "^-- REPLACE_RULES_SRC_TMP_FILE = $REPLACE_RULES_SRC_TMP_FILE" +echo "^-- REPLACE_RULES_DST_CP_FILE_TO = $REPLACE_RULES_DST_CP_FILE_TO" + + +# DEPLOYMENT +##### + +echo "^-- Perform rules.txt deployment (decompound-rules.txt eventually)" + +echo "^-- ... rules.txt" +python3 ./conf/push_common_rules.py $SRC_TMP_FILE "$SOLR_BASE_URL/$SOLR_COLLECTION_NAME/querqy/rewriter/$DST_CP_FILE_TO" +if [ $? -ne 0 ]; then + exit 16 +fi + +echo "^-- ... replace-rules.txt" +if ! [[ $REPLACE_RULES_SRC_TMP_FILE == "NONE" && $REPLACE_RULES_DST_CP_FILE_TO == "NONE" ]] +then + python3 /smui/conf/push_replace.py $REPLACE_RULES_SRC_TMP_FILE "$SOLR_BASE_URL/$SOLR_COLLECTION_NAME/querqy/rewriter/$REPLACE_RULES_DST_CP_FILE_TO" +fi + +# all ok +echo "smui2solrcloud.sh - ok" +exit 0 From 0402b08f63286137e2398af159ee9c6e123a64d5 Mon Sep 17 00:00:00 2001 From: paulb Date: Tue, 23 Nov 2021 17:08:03 +0100 Subject: [PATCH 3/5] Restore original function to avoid change of tests; this function now maps to first file in list (=default no tag export) --- app/controllers/ApiController.scala | 2 +- app/services/RulesTxtDeploymentService.scala | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/controllers/ApiController.scala b/app/controllers/ApiController.scala index eab4b2b4..557cd707 100644 --- a/app/controllers/ApiController.scala +++ b/app/controllers/ApiController.scala @@ -241,7 +241,7 @@ class ApiController @Inject()(authActionFactory: AuthActionFactory, logger.debug("In ApiController :: updateRulesTxtForSolrIndexAndTargetPlatform") // generate rules.txt(s) - val rulesFilesList = rulesTxtDeploymentService.generateRulesTxtContentWithFilenames(SolrIndexId(solrIndexId), targetSystem) + val rulesFilesList = rulesTxtDeploymentService.generateRulesTxtContentWithFilenamesList(SolrIndexId(solrIndexId), targetSystem) var apiResult: ApiResult = null for (rulesFiles <- rulesFilesList) { diff --git a/app/services/RulesTxtDeploymentService.scala b/app/services/RulesTxtDeploymentService.scala index 737ff148..cfa0e9ac 100644 --- a/app/services/RulesTxtDeploymentService.scala +++ b/app/services/RulesTxtDeploymentService.scala @@ -36,7 +36,11 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT sourceFileName: String, destinationFileName: String) - def generateRulesTxtContentWithFilenames(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true): List[RulesTxtsForSolrIndex] = { + def generateRulesTxtContentWithFilenames(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true): RulesTxtsForSolrIndex = { + generateRulesTxtContentWithFilenamesList(solrIndexId, targetSystem, logDebug).head + } + + def generateRulesTxtContentWithFilenamesList(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true): List[RulesTxtsForSolrIndex] = { val filterTagName = appConfig.get[String]("smui2solr.deployment.tag.property") val inputTagValues: List[InputTag] = if (!filterTagName.isEmpty) @@ -347,7 +351,7 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT try { for (index <- searchManagementRepository.listAllSolrIndexes) { // TODO make targetSystem configurable from ApiController.downloadAllRulesTxtFiles ... go with "LIVE" from now (as there exist no different revisions of the search management content)! - val rulesList = generateRulesTxtContentWithFilenames(index.id, "LIVE", logDebug = false) + val rulesList = generateRulesTxtContentWithFilenamesList(index.id, "LIVE", logDebug = false) for (rules <- rulesList) { zipStream.putNextEntry(new ZipEntry(s"rules_${index.name}.txt")) zipStream.write(rules.regularRules.content.getBytes("UTF-8")) From 9bb192f6f294ef9fb810927e5724be7a4622976a Mon Sep 17 00:00:00 2001 From: paulb Date: Tue, 23 Nov 2021 19:47:47 +0100 Subject: [PATCH 4/5] Add test for inputTag export --- .../TestRulesTxtImportTenantTags.json | 14 +++ ...tDeploymentServiceConfigVariantsSpec.scala | 90 ++++++++++++++++++- 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 test/resources/TestRulesTxtImportTenantTags.json diff --git a/test/resources/TestRulesTxtImportTenantTags.json b/test/resources/TestRulesTxtImportTenantTags.json new file mode 100644 index 00000000..1b0548af --- /dev/null +++ b/test/resources/TestRulesTxtImportTenantTags.json @@ -0,0 +1,14 @@ +[ + { + "property": "tenant", + "value": "AA" + }, + { + "property": "tenant", + "value": "BB" + }, + { + "property": "tenant", + "value": "CC" + } +] \ No newline at end of file diff --git a/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala b/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala index c84eff2f..eb1e15b7 100644 --- a/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala +++ b/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala @@ -13,6 +13,7 @@ trait CommonRulesTxtDeploymentServiceConfigVariantsSpecBase extends ApplicationT // TODO maybe share those definitions / instructions with RulesTxtDeploymentServiceSpec as well? protected lazy val service = injector.instanceOf[RulesTxtDeploymentService] + protected lazy val rulesTxtImportService = injector.instanceOf[RulesTxtImportService] override protected lazy val activateSpelling = false @@ -388,5 +389,92 @@ class RulesTxtDeploymentGitTargetSpec extends FlatSpec with Matchers with Common } - } + +class RulesTxtOnlyDeploymentInputTagBasedSpec extends FlatSpec with Matchers with CommonRulesTxtDeploymentServiceConfigVariantsSpecBase { + + override protected lazy val additionalAppConfig = Seq( + "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", + "smui2solr.DST_CP_FILE_TO" -> "common_rules", + "toggle.rule-deployment.pre-live.present" -> true, + "smui2solr.deploy-prelive-fn-rules-txt" -> "common_rules", + "toggle.rule-tagging" -> true, + "toggle.predefined-tags-file" -> "./test/resources/TestRulesTxtImportTenantTags.json", + "smui2solr.deployment.tag.property" -> "tenant" + ) + + override protected def beforeAll(): Unit = { + super.beforeAll() + createTenantTaggedRules() + } + + protected def createTenantTaggedRules(): Unit = { + var rules: String = s"""tenantAA => + | DOWN(10): down_x + | @{ "tenant":["AA"]}@ + | + |tenantBB => + | DOWN(10): down_x + | @{ "tenant" : [ "BB" ] }@ + | + |tenantNoTag => + | DOWN(10): down_x + | + |tenantNone => + | DOWN(10): down_x + | @{ "tenant" : [ ] }@ + | + |tenantAB => + | DOWN(10): down_x + | @{ "tenant" : [ "AA", "BB" ] }@ + | + |""".stripMargin + val ( + retstatCountRulesTxtInputs, + retstatCountRulesTxtLinesSkipped, + retstatCountRulesTxtUnkownConvert, + retstatCountConsolidatedInputs, + retstatCountConsolidatedRules + ) = rulesTxtImportService.importFromFilePayload(rules, core1Id) + } + + protected def getExpectedResultsList: List[Map[String, Object]] = { + List( + Map("inputTerms" -> List("tenantNoTag", "tenantNone", "aerosmith"), "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", "destinationFileName" -> "common_rules"), + Map("inputTerms" -> List("tenantAA", "tenantAB"), "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt_AA.tmp", "destinationFileName" -> "common_rules_AA"), + Map("inputTerms" -> List("tenantAB", "tenantBB"), "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt_BB.tmp", "destinationFileName" -> "common_rules_BB"), + Map("inputTerms" -> List(), "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt_CC.tmp", "destinationFileName" -> "common_rules_CC") + ) + } + + def validateDeploymentDescriptor(deploymentDescriptor: service.RulesTxtsForSolrIndex, expectedResults: Map[String, Object]) = { + deploymentDescriptor.solrIndexId shouldBe core1Id + val inputTerms : List[String] = expectedResults("inputTerms").asInstanceOf[List[String]] + for (inputTerm <- inputTerms) { + deploymentDescriptor.regularRules.content should include (inputTerm) + } + deploymentDescriptor.regularRules.sourceFileName shouldBe expectedResults("sourceFileName") + deploymentDescriptor.regularRules.destinationFileName shouldBe expectedResults("destinationFileName") + deploymentDescriptor.replaceRules shouldBe None + deploymentDescriptor.decompoundRules shouldBe None + } + + "RulesTxtDeploymentService" should "provide only the (common) rules.txt for PRELIVE" in { + val deploymentDescriptorList = service.generateRulesTxtContentWithFilenamesList(core1Id, "PRELIVE", logDebug = false) + var i = 0 + for (deploymentDescriptor <- deploymentDescriptorList) { + validateDeploymentDescriptor(deploymentDescriptor, getExpectedResultsList(i)) + i += 1 + } + } + + "RulesTxtDeploymentService" should "provide only the (common) rules.txt for LIVE" in { + val deploymentDescriptorList = service.generateRulesTxtContentWithFilenamesList(core1Id, "LIVE", logDebug = false) + var i = 0 + for (deploymentDescriptor <- deploymentDescriptorList) { + validateDeploymentDescriptor(deploymentDescriptor, getExpectedResultsList(i)) + i += 1 + } + } + +} \ No newline at end of file From b4b43748d5a1a9b961dff4b50fcbfdacabfc3762 Mon Sep 17 00:00:00 2001 From: paulb Date: Thu, 25 Nov 2021 17:58:01 +0100 Subject: [PATCH 5/5] Refactor and add some comments --- app/models/SearchManagementRepository.scala | 18 ++++++- .../querqy/QuerqyRulesTxtGenerator.scala | 29 ++++++------ app/services/RulesTxtDeploymentService.scala | 47 ++++++++++--------- conf/application.conf | 4 +- ...tDeploymentServiceConfigVariantsSpec.scala | 32 ++++++++++--- 5 files changed, 83 insertions(+), 47 deletions(-) diff --git a/app/models/SearchManagementRepository.scala b/app/models/SearchManagementRepository.scala index 6f00c84b..d1fcfb8d 100644 --- a/app/models/SearchManagementRepository.scala +++ b/app/models/SearchManagementRepository.scala @@ -3,7 +3,6 @@ package models import java.io.FileInputStream import java.time.LocalDateTime import java.util.{Date, UUID} - import javax.inject.Inject import play.api.db.DBApi import anorm._ @@ -13,6 +12,8 @@ import models.spellings.{CanonicalSpelling, CanonicalSpellingId, CanonicalSpelli import models.eventhistory.{ActivityLog, ActivityLogEntry, InputEvent} import models.reports.{ActivityReport, DeploymentLog, RulesReport} +import scala.collection.mutable.ListBuffer + @javax.inject.Singleton class SearchManagementRepository @Inject()(dbapi: DBApi, toggleService: FeatureToggleService)(implicit ec: DatabaseExecutionContext) { @@ -56,6 +57,21 @@ class SearchManagementRepository @Inject()(dbapi: DBApi, toggleService: FeatureT InputTag.loadAll().filter(_.solrIndexId== Option(solrIndexId)).filter(_.property == Option(inputTagProperty)).toList } + def listAllInputTagValuesForInputTagProperty(solrIndexId: SolrIndexId, filterInputTagProperty: String): List[InputTag] = { + val inputTagValuesListBuffer: ListBuffer[InputTag] = ListBuffer() + // retrieve inputTags common to all solr indices + for (inputTagValue <- listInputTagValuesForSolrIndexAndInputTagProperty(null, filterInputTagProperty)) { + inputTagValuesListBuffer += inputTagValue + } + // retrieve inputTags dedicated for current solr index + for (inputTagValue <- listInputTagValuesForSolrIndexAndInputTagProperty(solrIndexId, filterInputTagProperty)) { + inputTagValuesListBuffer += inputTagValue + } + inputTagValuesListBuffer.toList + } + + + def addNewInputTag(inputTag: InputTag) = db.withConnection { implicit connection => InputTag.insert(inputTag) } diff --git a/app/models/querqy/QuerqyRulesTxtGenerator.scala b/app/models/querqy/QuerqyRulesTxtGenerator.scala index f3ade0e2..856ce5cd 100644 --- a/app/models/querqy/QuerqyRulesTxtGenerator.scala +++ b/app/models/querqy/QuerqyRulesTxtGenerator.scala @@ -130,17 +130,18 @@ class QuerqyRulesTxtGenerator @Inject()(searchManagementRepository: SearchManage retQuerqyRulesTxt.toString() } - def hasValidInputTagValue(i: SearchInputWithRules, filterTagName: String, inputTag: InputTag): Boolean = { - // no filter on tags defined - if (filterTagName == null || filterTagName.isEmpty) { + def hasValidInputTagValue(searchInputWithRules: SearchInputWithRules, filterInputTagProperty: String, currentInputTag: InputTag): Boolean = { + // no filtering on tags + if (filterInputTagProperty.isEmpty) { return true } - val ruleTags = i.tags.filter(_.property.get.equals(filterTagName)) - // common rules (without the input tag - if (inputTag == null) { - ruleTags.isEmpty + val filteredInputTagsOfCurrentRule = searchInputWithRules.tags.filter(_.property.get.equals(filterInputTagProperty)) + if (currentInputTag == null) { + // common set of rules (inputTag == null): check whether the filterInputTagProperty as property is absent + filteredInputTagsOfCurrentRule.isEmpty } else { - ruleTags.map(t => t.id).contains(inputTag.id) + // other rules sets per input tag value: check whether the current tag has been set on the rule + filteredInputTagsOfCurrentRule.map(t => t.id).contains(currentInputTag.id) } } @@ -153,7 +154,7 @@ class QuerqyRulesTxtGenerator @Inject()(searchManagementRepository: SearchManage * @return */ // TODO resolve & test logic of render method (change interface to separate decompound from normal rules) - private def render(solrIndexId: SolrIndexId, separateRulesTxts: Boolean, renderCompoundsRulesTxt: Boolean, filterTagName: String, inputTag: InputTag): String = { + private def render(solrIndexId: SolrIndexId, separateRulesTxts: Boolean, renderCompoundsRulesTxt: Boolean, filterInputTagProperty: String, currentInputTag: InputTag): String = { val retQuerqyRulesTxt = new StringBuilder() @@ -164,7 +165,7 @@ class QuerqyRulesTxtGenerator @Inject()(searchManagementRepository: SearchManage .filter(i => i.trimmedTerm.nonEmpty) // TODO it needs to be ensured, that a rule not only exists in the list, are active, BUT also has a filled term (after trim) .filter(_.hasAnyActiveRules) - .filter(i => hasValidInputTagValue(i, filterTagName, inputTag)) + .filter(searchInputWithRules => hasValidInputTagValue(searchInputWithRules, filterInputTagProperty, currentInputTag)) // TODO merge decompound identification login with ApiController :: validateSearchInputToErrMsg @@ -184,12 +185,12 @@ class QuerqyRulesTxtGenerator @Inject()(searchManagementRepository: SearchManage renderListSearchInputRules(separateRules(listSearchInput)) } - def renderSingleRulesTxt(solrIndexId: SolrIndexId, filterTagName: String, inputTag: InputTag): String = { - render(solrIndexId, false, false, filterTagName, inputTag) + def renderSingleRulesTxt(solrIndexId: SolrIndexId, filterInputTagProperty: String, currentInputTag: InputTag): String = { + render(solrIndexId, false, false, filterInputTagProperty, currentInputTag) } - def renderSeparatedRulesTxts(solrIndexId: SolrIndexId, renderCompoundsRulesTxt: Boolean, filterTagName: String, inputTag: InputTag): String = { - render(solrIndexId, true, renderCompoundsRulesTxt, filterTagName, inputTag) + def renderSeparatedRulesTxts(solrIndexId: SolrIndexId, renderCompoundsRulesTxt: Boolean, filterInputTagProperty: String, currentInputTag: InputTag): String = { + render(solrIndexId, true, renderCompoundsRulesTxt, filterInputTagProperty, currentInputTag) } /** diff --git a/app/services/RulesTxtDeploymentService.scala b/app/services/RulesTxtDeploymentService.scala index cfa0e9ac..e2f244bd 100644 --- a/app/services/RulesTxtDeploymentService.scala +++ b/app/services/RulesTxtDeploymentService.scala @@ -1,15 +1,14 @@ package services -import java.io.OutputStream -import java.util.zip.{ZipEntry, ZipOutputStream} -import javax.inject.Inject import models.FeatureToggleModel.FeatureToggleService import models.input.InputTag import models.querqy.{QuerqyReplaceRulesGenerator, QuerqyRulesTxtGenerator} import models.{DeploymentScriptResult, SearchManagementRepository, SolrIndexId} import play.api.{Configuration, Environment, Logging} -import scala.collection.immutable.HashSet +import java.io.OutputStream +import java.util.zip.{ZipEntry, ZipOutputStream} +import javax.inject.Inject import scala.collection.mutable.ListBuffer import scala.sys.process._ @@ -36,22 +35,23 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT sourceFileName: String, destinationFileName: String) + // for backward compatibility: returns first element from generateRulesTxtContentWithFilenamesList(...) def generateRulesTxtContentWithFilenames(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true): RulesTxtsForSolrIndex = { generateRulesTxtContentWithFilenamesList(solrIndexId, targetSystem, logDebug).head } + // generate list of files per intputTag value (including the common set without the tag property) def generateRulesTxtContentWithFilenamesList(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true): List[RulesTxtsForSolrIndex] = { - val filterTagName = appConfig.get[String]("smui2solr.deployment.tag.property") - val inputTagValues: List[InputTag] = - if (!filterTagName.isEmpty) - searchManagementRepository.listInputTagValuesForSolrIndexAndInputTagProperty(null, filterTagName) ++ searchManagementRepository.listInputTagValuesForSolrIndexAndInputTagProperty(solrIndexId, filterTagName) - else - List.empty + val filterInputTagProperty = appConfig.get[String]("smui2solr.deployment.per.inputtag.property") + // build rules list based on the filter val rulesTxtsForSolrIndexes: ListBuffer[RulesTxtsForSolrIndex] = new ListBuffer[RulesTxtsForSolrIndex] - // first rules without the inputTag assigned - rulesTxtsForSolrIndexes += generateRulesTxtContentWithFilenamesWithTags(solrIndexId, targetSystem, true, filterTagName, null) - for (i <- inputTagValues) { - rulesTxtsForSolrIndexes += generateRulesTxtContentWithFilenamesWithTags(solrIndexId, targetSystem, true, filterTagName, i) + // 1st: common rules without the inputTag assigned + rulesTxtsForSolrIndexes += generateRulesTxtContentWithFilenamesPerInputTag(solrIndexId, targetSystem, true, filterInputTagProperty, null) + // 2nd: rules per inputTag values + if (!filterInputTagProperty.isEmpty) { + for (currentInputTag <- searchManagementRepository.listAllInputTagValuesForInputTagProperty(solrIndexId, filterInputTagProperty)) { + rulesTxtsForSolrIndexes += generateRulesTxtContentWithFilenamesPerInputTag(solrIndexId, targetSystem, true, filterInputTagProperty, currentInputTag) + } } rulesTxtsForSolrIndexes.toList } @@ -62,10 +62,9 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT * @param solrIndexId Solr Index Id to generate the output for. */ // TODO evaluate, if logDebug should be used to prevent verbose logging of the whole generated rules.txt (for zip download especially) - def generateRulesTxtContentWithFilenamesWithTags(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true, filterTagName: String, inputTag: InputTag): RulesTxtsForSolrIndex = { - - val inputTagValue:String = if (inputTag != null) - inputTag.value + def generateRulesTxtContentWithFilenamesPerInputTag(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true, filterInputTagProperty: String, currentInputTag: InputTag): RulesTxtsForSolrIndex = { + val inputTagValue:String = if (currentInputTag != null) + currentInputTag.value else "" // SMUI config for (regular) LIVE deployment @@ -113,8 +112,9 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT val sourceTempFile = SRC_TMP_FILE - val replaceRules = - if (EXPORT_REPLACE_RULES && inputTag == null) { + val replaceRules = { + // exclude iterations for inputtag value != null as replace rules have no inputtags + if (EXPORT_REPLACE_RULES && currentInputTag == null) { val allCanonicalSpellings = searchManagementRepository.listAllSpellingsWithAlternatives(solrIndexId) Some(RulesTxtWithFileNames( QuerqyReplaceRulesGenerator.renderAllCanonicalSpellingsToReplaceRules(allCanonicalSpellings), @@ -122,10 +122,11 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT replaceRulesDstCpFileTo )) } else None + } if (!DO_SPLIT_DECOMPOUND_RULES_TXT) { RulesTxtsForSolrIndex(solrIndexId, - RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSingleRulesTxt(solrIndexId, filterTagName, inputTag), sourceTempFile, dstCpFileTo), + RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSingleRulesTxt(solrIndexId, filterInputTagProperty, currentInputTag), sourceTempFile, dstCpFileTo), None, replaceRules ) @@ -135,8 +136,8 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT else // targetSystem == "LIVE" DECOMPOUND_RULES_TXT_DST_CP_FILE_TO RulesTxtsForSolrIndex(solrIndexId, - RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = false, filterTagName, inputTag), sourceTempFile, dstCpFileTo), - Some(RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = true, filterTagName, inputTag), + RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = false, filterInputTagProperty, currentInputTag), sourceTempFile, dstCpFileTo), + Some(RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = true, filterInputTagProperty, currentInputTag), sourceTempFile + "-2", decompoundDstCpFileTo) ), replaceRules diff --git a/conf/application.conf b/conf/application.conf index 7bae621e..58681b25 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -35,8 +35,8 @@ smui.deployment.git.repo-url=${?SMUI_DEPLOYMENT_GIT_REPO_URL} smui2solr.deployment.git.filename.common-rules-txt="rules.txt" smui2solr.deployment.git.filename.common-rules-txt=${?SMUI_DEPLOYMENT_GIT_FN_COMMON_RULES_TXT} -smui2solr.deployment.tag.property="" -smui2solr.deployment.tag.property=${?SMUI_2SOLR_DEPLOYMENT_BY_TAG} +smui2solr.deployment.per.inputtag.property="" +smui2solr.deployment.per.inputtag.property=${?SMUI_2SOLR_DEPLOYMENT_PER_INPUTTAG_PROPERTY} # Application Feature Toggles # ~~~~~ diff --git a/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala b/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala index eb1e15b7..06f15d29 100644 --- a/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala +++ b/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala @@ -400,7 +400,7 @@ class RulesTxtOnlyDeploymentInputTagBasedSpec extends FlatSpec with Matchers wit "smui2solr.deploy-prelive-fn-rules-txt" -> "common_rules", "toggle.rule-tagging" -> true, "toggle.predefined-tags-file" -> "./test/resources/TestRulesTxtImportTenantTags.json", - "smui2solr.deployment.tag.property" -> "tenant" + "smui2solr.deployment.per.inputtag.property" -> "tenant" ) override protected def beforeAll(): Unit = { @@ -440,19 +440,37 @@ class RulesTxtOnlyDeploymentInputTagBasedSpec extends FlatSpec with Matchers wit protected def getExpectedResultsList: List[Map[String, Object]] = { List( - Map("inputTerms" -> List("tenantNoTag", "tenantNone", "aerosmith"), "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", "destinationFileName" -> "common_rules"), - Map("inputTerms" -> List("tenantAA", "tenantAB"), "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt_AA.tmp", "destinationFileName" -> "common_rules_AA"), - Map("inputTerms" -> List("tenantAB", "tenantBB"), "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt_BB.tmp", "destinationFileName" -> "common_rules_BB"), - Map("inputTerms" -> List(), "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt_CC.tmp", "destinationFileName" -> "common_rules_CC") + // common - without tenant inputTag value + Map("inputTermsIncl" -> List("tenantNoTag", "tenantNone", "aerosmith"), + "inputTermsExcl" -> List("tenantAA", "tenantAB", "tenantBB"), + "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", + "destinationFileName" -> "common_rules"), + // tenant AA + Map("inputTermsIncl" -> List("tenantAA", "tenantAB"), + "inputTermsExcl" -> List("tenantNoTag", "tenantNone", "aerosmith", "tenantBB"), + "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt_AA.tmp", + "destinationFileName" -> "common_rules_AA"), + // tenant BB + Map("inputTermsIncl" -> List("tenantAB", "tenantBB"), + "inputTermsExcl" -> List("tenantNoTag", "tenantNone", "aerosmith", "tenantAA"), + "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt_BB.tmp", + "destinationFileName" -> "common_rules_BB"), + // tenant CC + Map("inputTermsIncl" -> List(), + "inputTermsExcl" -> List("tenantNoTag", "tenantNone", "aerosmith", "tenantAA", "tenantAB", "tenantBB"), + "sourceFileName" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt_CC.tmp", + "destinationFileName" -> "common_rules_CC") ) } def validateDeploymentDescriptor(deploymentDescriptor: service.RulesTxtsForSolrIndex, expectedResults: Map[String, Object]) = { deploymentDescriptor.solrIndexId shouldBe core1Id - val inputTerms : List[String] = expectedResults("inputTerms").asInstanceOf[List[String]] - for (inputTerm <- inputTerms) { + for (inputTerm <- expectedResults("inputTermsIncl").asInstanceOf[List[String]]) { deploymentDescriptor.regularRules.content should include (inputTerm) } + for (inputTerm <- expectedResults("inputTermsExcl").asInstanceOf[List[String]]) { + deploymentDescriptor.regularRules.content should not include (inputTerm) + } deploymentDescriptor.regularRules.sourceFileName shouldBe expectedResults("sourceFileName") deploymentDescriptor.regularRules.destinationFileName shouldBe expectedResults("destinationFileName") deploymentDescriptor.replaceRules shouldBe None