From 3e68d98d7844208ce179957ee515c1c525c3b4f5 Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Sun, 7 Oct 2018 02:33:53 +0900 Subject: [PATCH 1/9] Change the digdag badge: use v0.9.30 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fde026d..93e6295 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # digdag-operator-param -[![Jitpack](https://jitpack.io/v/pro.civitaspo/digdag-operator-param.svg)](https://jitpack.io/#pro.civitaspo/digdag-operator-param) [![CircleCI](https://circleci.com/gh/civitaspo/digdag-operator-param.svg?style=shield)](https://circleci.com/gh/civitaspo/digdag-operator-param) [![Digdag](https://img.shields.io/badge/digdag-v0.9.27-brightgreen.svg)](https://github.com/treasure-data/digdag/releases/tag/v0.9.28) +[![Jitpack](https://jitpack.io/v/pro.civitaspo/digdag-operator-param.svg)](https://jitpack.io/#pro.civitaspo/digdag-operator-param) [![CircleCI](https://circleci.com/gh/civitaspo/digdag-operator-param.svg?style=shield)](https://circleci.com/gh/civitaspo/digdag-operator-param) [![Digdag](https://img.shields.io/badge/digdag-v0.9.30-brightgreen.svg)](https://github.com/treasure-data/digdag/releases/tag/v0.9.30) digdag plugin for operating params. From fc3fa1733cf4bc46e649476a041cb0f75f2b23d8 Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Tue, 23 Oct 2018 20:32:13 +0900 Subject: [PATCH 2/9] Add TemplateEngine and ConfigFactory to Abstract class for ParamEvalOperator --- .../pro/civitaspo/digdag/plugin/param/ParamPlugin.scala | 9 ++++++--- .../plugin/param/operator/AbstractParamOperator.scala | 7 ++++--- .../plugin/param/operator/ParamResetOperator.scala | 4 ++-- .../plugin/param/operator/ParamStoreOperator.scala | 4 ++-- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/param/ParamPlugin.scala b/src/main/scala/pro/civitaspo/digdag/plugin/param/ParamPlugin.scala index bd0f275..bbb53fe 100644 --- a/src/main/scala/pro/civitaspo/digdag/plugin/param/ParamPlugin.scala +++ b/src/main/scala/pro/civitaspo/digdag/plugin/param/ParamPlugin.scala @@ -3,13 +3,16 @@ package pro.civitaspo.digdag.plugin.param import java.lang.reflect.Constructor import java.util.{Arrays => JArrays, List => JList} -import io.digdag.spi.{Operator, OperatorContext, OperatorFactory, OperatorProvider, Plugin} +import io.digdag.spi.{Operator, OperatorContext, OperatorFactory, OperatorProvider, Plugin, TemplateEngine} +import javax.inject.Inject import pro.civitaspo.digdag.plugin.param.operator.{AbstractParamOperator, ParamResetOperator, ParamStoreOperator} object ParamPlugin { class AthenaOperatorProvider extends OperatorProvider { + @Inject protected var templateEngine: TemplateEngine = null + override def get(): JList[OperatorFactory] = { JArrays.asList(operatorFactory("param_reset", classOf[ParamResetOperator]), operatorFactory("param_store", classOf[ParamStoreOperator])) } @@ -18,8 +21,8 @@ object ParamPlugin { new OperatorFactory { override def getType: String = operatorName override def newOperator(context: OperatorContext): Operator = { - val constructor: Constructor[T] = klass.getConstructor(classOf[String], classOf[OperatorContext]) - constructor.newInstance(operatorName, context) + val constructor: Constructor[T] = klass.getConstructor(classOf[String], classOf[OperatorContext], classOf[TemplateEngine]) + constructor.newInstance(operatorName, context, templateEngine) } } } diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/AbstractParamOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/AbstractParamOperator.scala index c6ba46a..eb9f12b 100644 --- a/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/AbstractParamOperator.scala +++ b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/AbstractParamOperator.scala @@ -1,13 +1,14 @@ package pro.civitaspo.digdag.plugin.param.operator -import io.digdag.client.config.Config -import io.digdag.spi.OperatorContext +import io.digdag.client.config.{Config, ConfigFactory} +import io.digdag.spi.{OperatorContext, TemplateEngine} import io.digdag.util.BaseOperator import org.slf4j.{Logger, LoggerFactory} -abstract class AbstractParamOperator(operatorName: String, context: OperatorContext) extends BaseOperator(context) { +abstract class AbstractParamOperator(operatorName: String, context: OperatorContext, templateEngine: TemplateEngine) extends BaseOperator(context) { protected val logger: Logger = LoggerFactory.getLogger(operatorName) + protected val cf: ConfigFactory = request.getConfig.getFactory protected val params: Config = request.getConfig } diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamResetOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamResetOperator.scala index b4f494f..7c42adf 100644 --- a/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamResetOperator.scala +++ b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamResetOperator.scala @@ -2,9 +2,9 @@ package pro.civitaspo.digdag.plugin.param.operator import com.google.common.collect.ImmutableList import io.digdag.client.config.ConfigKey -import io.digdag.spi.{OperatorContext, TaskResult} +import io.digdag.spi.{OperatorContext, TaskResult, TemplateEngine} -class ParamResetOperator(operatorName: String, context: OperatorContext) extends AbstractParamOperator(operatorName, context) { +class ParamResetOperator(operatorName: String, context: OperatorContext, templateEngine: TemplateEngine) extends AbstractParamOperator(operatorName, context, templateEngine) { protected val resetKey: String = params.get("_command", classOf[String]) diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamStoreOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamStoreOperator.scala index c84294b..ea59fc7 100644 --- a/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamStoreOperator.scala +++ b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamStoreOperator.scala @@ -1,9 +1,9 @@ package pro.civitaspo.digdag.plugin.param.operator import io.digdag.client.config.Config -import io.digdag.spi.{OperatorContext, TaskResult} +import io.digdag.spi.{OperatorContext, TaskResult, TemplateEngine} -class ParamStoreOperator(operatorName: String, context: OperatorContext) extends AbstractParamOperator(operatorName, context) { +class ParamStoreOperator(operatorName: String, context: OperatorContext, templateEngine: TemplateEngine) extends AbstractParamOperator(operatorName, context, templateEngine) { protected val newParams: Config = params.get("_command", classOf[Config]) From fcd083f21a0062669f4351c1e9cd564b3eed0204 Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Tue, 23 Oct 2018 20:33:45 +0900 Subject: [PATCH 3/9] Add ParamEvalOperator --- .../digdag/plugin/param/ParamPlugin.scala | 8 +++- .../param/operator/ParamEvalOperator.scala | 46 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamEvalOperator.scala diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/param/ParamPlugin.scala b/src/main/scala/pro/civitaspo/digdag/plugin/param/ParamPlugin.scala index bbb53fe..3297486 100644 --- a/src/main/scala/pro/civitaspo/digdag/plugin/param/ParamPlugin.scala +++ b/src/main/scala/pro/civitaspo/digdag/plugin/param/ParamPlugin.scala @@ -5,7 +5,7 @@ import java.util.{Arrays => JArrays, List => JList} import io.digdag.spi.{Operator, OperatorContext, OperatorFactory, OperatorProvider, Plugin, TemplateEngine} import javax.inject.Inject -import pro.civitaspo.digdag.plugin.param.operator.{AbstractParamOperator, ParamResetOperator, ParamStoreOperator} +import pro.civitaspo.digdag.plugin.param.operator.{AbstractParamOperator, ParamEvalOperator, ParamResetOperator, ParamStoreOperator} object ParamPlugin { @@ -14,7 +14,11 @@ object ParamPlugin { @Inject protected var templateEngine: TemplateEngine = null override def get(): JList[OperatorFactory] = { - JArrays.asList(operatorFactory("param_reset", classOf[ParamResetOperator]), operatorFactory("param_store", classOf[ParamStoreOperator])) + JArrays.asList( + operatorFactory("param_reset", classOf[ParamResetOperator]), + operatorFactory("param_store", classOf[ParamStoreOperator]), + operatorFactory("param_eval", classOf[ParamEvalOperator]) + ) } private def operatorFactory[T <: AbstractParamOperator](operatorName: String, klass: Class[T]): OperatorFactory = { diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamEvalOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamEvalOperator.scala new file mode 100644 index 0000000..32c92be --- /dev/null +++ b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamEvalOperator.scala @@ -0,0 +1,46 @@ +package pro.civitaspo.digdag.plugin.param.operator +import java.nio.charset.StandardCharsets.UTF_8 + +import com.google.common.collect.ImmutableList +import io.digdag.client.config.{Config, ConfigKey} +import io.digdag.spi.{OperatorContext, TaskResult, TemplateEngine} + +class ParamEvalOperator(operatorName: String, context: OperatorContext, templateEngine: TemplateEngine) + extends AbstractParamOperator(operatorName, context, templateEngine) { + + protected val key: String = params.get("_command", classOf[String]) + + override def runTask(): TaskResult = { + val evaluated: Config = eval(params) + val paramsToStore: Config = cf.create() + + val elems: Seq[String] = key.split("\\.") + val parents: Seq[String] = elems.reverse.tail.reverse + val child: String = elems.last + + if (parents.isEmpty) paramsToStore.set(child, evaluated.get(child, classOf[Object])) + else { + val getter = parents.foldLeft(evaluated) { (nested: Config, k: String) => nested.getNested(k) + } + val setter = parents.foldLeft(paramsToStore) { (nested: Config, k: String) => nested.getNestedOrSetEmpty(k) + } + setter.set(child, getter.get(child, classOf[Object])) + } + + val builder = TaskResult.defaultBuilder(cf) + builder.resetStoreParams(ImmutableList.of(ConfigKey.parse(key))) + builder.storeParams(paramsToStore) + builder.build() + } + + protected def eval(params: Config): Config = { + val tmpFile: String = workspace.createTempFile("param_eval", ".json") + + val writer = workspace.newBufferedWriter(tmpFile, UTF_8) + try writer.write(params.toString) + finally writer.close() + + val content = workspace.templateFile(templateEngine, tmpFile, UTF_8, params) + cf.fromJsonString(content) + } +} From d4e0785c2d962ac4f68d0852c79f12c1c147edcf Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Tue, 23 Oct 2018 20:34:22 +0900 Subject: [PATCH 4/9] Add examples for param_eval> operator --- example/example.dig | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/example/example.dig b/example/example.dig index 102590b..c6e8ead 100644 --- a/example/example.dig +++ b/example/example.dig @@ -27,3 +27,34 @@ _export: +show3: echo>: "hoge: ${typeof(hoge) == 'undefined' ? 'None' : hoge}, a.b: ${typeof(a) == 'undefined' ? 'None' : typeof(a.b) == 'undefined' ? 'None' : a.b}" ++eval: + _export: + a: aaa + b: bbb + c: ccc + d: ${a}-${b}-${c} + + +a: + _export: + e: ${d} + f: + g: ${d} + h: + - ${d} + +b: + echo>: ${d} + +c: + echo>: ${e} + +d: + echo>: ${f.g} + +e: + param_eval>: f.g + +f: + echo>: ${f.g} + +g: + echo>: ${f.h} + +h: + param_eval>: f.h + +i: + echo>: ${f.h} + From 2e69707154ffedaff63ad10b76140ce83db3d669 Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Tue, 23 Oct 2018 20:37:53 +0900 Subject: [PATCH 5/9] Write README --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 93e6295..7f70e27 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,13 @@ _export: - **param_reset>**: Param name to reset. (string, required) +## Configuration for `param_eval>` operator + +### Options + +- **param_eval>**: Param name to eval. (string, required) + - **NOTE**: This operator is a workaround for the issue: [Exported vars are not evaluated recursively in the context of nested params](https://github.com/treasure-data/digdag/issues/862) + # Development ## Run an Example From 69bb6a5b5b2066e19e9eb3ffcde688a67a458d2a Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Tue, 23 Oct 2018 20:45:01 +0900 Subject: [PATCH 6/9] More examples for param_eval> --- example/example.dig | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/example/example.dig b/example/example.dig index c6e8ead..0fee7af 100644 --- a/example/example.dig +++ b/example/example.dig @@ -52,9 +52,19 @@ _export: +f: echo>: ${f.g} +g: - echo>: ${f.h} - +h: + +h: + for_each>: {i: "${f.h}"} + _do: + echo>: ${i} + +i: + echo>: ${f.h} + +j: param_eval>: f.h - +i: - echo>: ${f.h} + +k: + +l: + for_each>: {i: "${f.h}"} + _do: + echo>: ${i} + +m: + echo>: ${f.h} From 0e458fded710919fc81a9d2c4923ac742cac7758 Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Tue, 23 Oct 2018 20:47:59 +0900 Subject: [PATCH 7/9] Add usage for param_eval operator --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/README.md b/README.md index 7f70e27..41139f3 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,47 @@ _export: +show3: echo>: "hoge: ${typeof(hoge) == 'undefined' ? 'None' : hoge}, a.b: ${typeof(a) == 'undefined' ? 'None' : typeof(a.b) == 'undefined' ? 'None' : a.b}" ++eval: + _export: + a: aaa + b: bbb + c: ccc + d: ${a}-${b}-${c} + + +a: + _export: + e: ${d} + f: + g: ${d} + h: + - ${d} + +b: + echo>: ${d} + +c: + echo>: ${e} + +d: + echo>: ${f.g} + +e: + param_eval>: f.g + +f: + echo>: ${f.g} + +g: + +h: + for_each>: {i: "${f.h}"} + _do: + echo>: ${i} + +i: + echo>: ${f.h} + +j: + param_eval>: f.h + +k: + +l: + for_each>: {i: "${f.h}"} + _do: + echo>: ${i} + +m: + echo>: ${f.h} + ``` # Configuration From d86c39d5a951fbb36cc361b47c6e61f66df53419 Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Tue, 23 Oct 2018 20:48:39 +0900 Subject: [PATCH 8/9] ./gradlew spotlessapply --- .../digdag/plugin/param/operator/ParamEvalOperator.scala | 6 ++++-- .../digdag/plugin/param/operator/ParamResetOperator.scala | 3 ++- .../digdag/plugin/param/operator/ParamStoreOperator.scala | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamEvalOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamEvalOperator.scala index 32c92be..056b74f 100644 --- a/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamEvalOperator.scala +++ b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamEvalOperator.scala @@ -20,9 +20,11 @@ class ParamEvalOperator(operatorName: String, context: OperatorContext, template if (parents.isEmpty) paramsToStore.set(child, evaluated.get(child, classOf[Object])) else { - val getter = parents.foldLeft(evaluated) { (nested: Config, k: String) => nested.getNested(k) + val getter = parents.foldLeft(evaluated) { (nested: Config, k: String) => + nested.getNested(k) } - val setter = parents.foldLeft(paramsToStore) { (nested: Config, k: String) => nested.getNestedOrSetEmpty(k) + val setter = parents.foldLeft(paramsToStore) { (nested: Config, k: String) => + nested.getNestedOrSetEmpty(k) } setter.set(child, getter.get(child, classOf[Object])) } diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamResetOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamResetOperator.scala index 7c42adf..42fa291 100644 --- a/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamResetOperator.scala +++ b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamResetOperator.scala @@ -4,7 +4,8 @@ import com.google.common.collect.ImmutableList import io.digdag.client.config.ConfigKey import io.digdag.spi.{OperatorContext, TaskResult, TemplateEngine} -class ParamResetOperator(operatorName: String, context: OperatorContext, templateEngine: TemplateEngine) extends AbstractParamOperator(operatorName, context, templateEngine) { +class ParamResetOperator(operatorName: String, context: OperatorContext, templateEngine: TemplateEngine) + extends AbstractParamOperator(operatorName, context, templateEngine) { protected val resetKey: String = params.get("_command", classOf[String]) diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamStoreOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamStoreOperator.scala index ea59fc7..a77d042 100644 --- a/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamStoreOperator.scala +++ b/src/main/scala/pro/civitaspo/digdag/plugin/param/operator/ParamStoreOperator.scala @@ -3,7 +3,8 @@ package pro.civitaspo.digdag.plugin.param.operator import io.digdag.client.config.Config import io.digdag.spi.{OperatorContext, TaskResult, TemplateEngine} -class ParamStoreOperator(operatorName: String, context: OperatorContext, templateEngine: TemplateEngine) extends AbstractParamOperator(operatorName, context, templateEngine) { +class ParamStoreOperator(operatorName: String, context: OperatorContext, templateEngine: TemplateEngine) + extends AbstractParamOperator(operatorName, context, templateEngine) { protected val newParams: Config = params.get("_command", classOf[Config]) From bb1d29ca103673f5231a15006f9e1663a18e9f8f Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Tue, 23 Oct 2018 20:52:07 +0900 Subject: [PATCH 9/9] Ship v0.0.2 --- CHANGELOG.md | 5 +++++ README.md | 2 +- build.gradle | 2 +- example/example.dig | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03d54c1..5685780 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +0.0.2 (2018-10-23) +================== + +* [Feature] Add `param_eval>` operator that is a workaround for https://github.com/treasure-data/digdag/issues/862 + 0.0.1 (2018-10-07) ================== diff --git a/README.md b/README.md index 41139f3..3507a0c 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ _export: repositories: - https://jitpack.io dependencies: - - pro.civitaspo:digdag-operator-param:0.0.1 + - pro.civitaspo:digdag-operator-param:0.0.2 +show1: echo>: "hoge: ${typeof(hoge) == 'undefined' ? 'None' : hoge}, a.b: ${typeof(a) == 'undefined' ? 'None' : typeof(a.b) == 'undefined' ? 'None' : a.b}" diff --git a/build.gradle b/build.gradle index ac8dccf..0d5fec2 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ plugins { } group = 'pro.civitaspo' -version = '0.0.1' +version = '0.0.2' def digdagVersion = '0.9.30' def scalaSemanticVersion = "2.12.6" diff --git a/example/example.dig b/example/example.dig index 0fee7af..1cdf025 100644 --- a/example/example.dig +++ b/example/example.dig @@ -4,7 +4,7 @@ _export: - file://${repos} # - https://jitpack.io dependencies: - - pro.civitaspo:digdag-operator-param:0.0.1 + - pro.civitaspo:digdag-operator-param:0.0.2 +show1: echo>: "hoge: ${typeof(hoge) == 'undefined' ? 'None' : hoge}, a.b: ${typeof(a) == 'undefined' ? 'None' : typeof(a.b) == 'undefined' ? 'None' : a.b}"