diff --git a/digdag-core/src/test/resources/io/digdag/core/workflow/multiple_operators.dig b/digdag-core/src/test/resources/io/digdag/core/workflow/multiple_operators.dig index 70a4b89f59..45d0bf32fb 100644 --- a/digdag-core/src/test/resources/io/digdag/core/workflow/multiple_operators.dig +++ b/digdag-core/src/test/resources/io/digdag/core/workflow/multiple_operators.dig @@ -1,3 +1,4 @@ +step1: sh>: echo this should fail py>: bar + python: python3 diff --git a/digdag-standards/src/main/resources/digdag/standards/py/runner.py b/digdag-standards/src/main/resources/digdag/standards/py/runner.py index 3743cc9f1e..4227ba3b17 100644 --- a/digdag-standards/src/main/resources/digdag/standards/py/runner.py +++ b/digdag-standards/src/main/resources/digdag/standards/py/runner.py @@ -49,12 +49,7 @@ def store(self, params={}, **kwds): def add_subtask(self, function=None, **params): if function is not None and not isinstance(function, dict): - if hasattr(function, "im_class"): - # Python 2 - command = ".".join([function.im_class.__module__, function.im_class.__name__, function.__name__]) - else: - # Python 3 - command = ".".join([function.__module__, function.__qualname__]) + command = ".".join([function.__module__, function.__qualname__]) config = params config["py>"] = command else: @@ -66,7 +61,7 @@ def add_subtask(self, function=None, **params): try: json.dumps(config) except Exception as error: - raise TypeError("Parameters must be serializable using JSON: %s" % str(error)) + raise TypeError(f"Parameters must be serializable using JSON: {str(error)}") self.subtask_config["+subtask" + str(self.subtask_index)] = config self.subtask_index += 1 @@ -87,14 +82,14 @@ def digdag_inspect_command(command): try: callable_type = getattr(mod, method_name) except AttributeError as error: - raise AttributeError("Module '%s' has no attribute '%s'" % (".".join(fragments), method_name)) + raise AttributeError(f"Module '{'.'.join(fragments)}' has no attribute '{method_name}'") except ImportError as error: class_name = fragments.pop() mod = __import__(".".join(fragments), fromlist=[class_name]) try: class_type = getattr(mod, class_name) except AttributeError as error: - raise AttributeError("Module '%s' has no attribute '%s'" % (".".join(fragments), method_name)) + raise AttributeError(f"Module '{'.'.join(fragments)}' has no attribute '{method_name}'") if type(callable_type) == type: class_type = callable_type @@ -109,12 +104,8 @@ def digdag_inspect_arguments(callable_type, exclude_self, params): if callable_type == object.__init__: # object.__init__ accepts *varargs and **keywords but it throws exception return {} - if hasattr(inspect, 'getfullargspec'): # Python3 - spec = inspect.getfullargspec(callable_type) - keywords_ = spec.varkw - else: # Python 2 - spec = inspect.getargspec(callable_type) - keywords_ = spec.keywords + spec = inspect.getfullargspec(callable_type) + keywords_ = spec.varkw args = {} for idx, key in enumerate(spec.args): @@ -125,15 +116,8 @@ def digdag_inspect_arguments(callable_type, exclude_self, params): else: if spec.defaults is None or idx < len(spec.args) - len(spec.defaults): # this keyword is required but not in params. raising an error. - if hasattr(callable_type, '__qualname__'): - # Python 3 - name = callable_type.__qualname__ - elif hasattr(callable_type, 'im_class'): - # Python 2 - name = "%s.%s" % (callable_type.im_class.__name__, callable_type.__name__) - else: - name = callable_type.__name__ - raise TypeError("Method '%s' requires parameter '%s' but not set" % (name, key)) + name = callable_type.__qualname__ + raise TypeError(f"Method '{name}' requires parameter '{key}' but not set") if keywords_: # above code was only for validation return params @@ -162,9 +146,9 @@ def digdag_inspect_arguments(callable_type, exclude_self, params): except SystemExit as e: # SystemExit only shows an exit code and it is not kind to users. So this block creates a specific error message. # This error will happen if called python module name and method name are equal to those of the standard library module. (e.g. tokenize.main) - error = Exception("Failed to call python command with code:%d" % e.code, "Possible cause: Invalid python module call, duplicate module name with standard library") + error = Exception(f"Failed to call python command with code:{str(e.code)}", "Possible cause: Ivalid python module call, duplicate module name with standard library") error_type, error_value, _tb = sys.exc_info() - error_message = "%s %s" % (error.args[0], error.args[1]) + error_message = " ".join(error.args[:2]) error_traceback = traceback.format_exception(error_type, error_value, _tb) except Exception as e: error = e diff --git a/digdag-tests/src/test/java/acceptance/PyIT.java b/digdag-tests/src/test/java/acceptance/PyIT.java index bfaa846c17..31d5dd152c 100644 --- a/digdag-tests/src/test/java/acceptance/PyIT.java +++ b/digdag-tests/src/test/java/acceptance/PyIT.java @@ -100,8 +100,8 @@ public void verifyConfigurationPythonOption() assertThat(attempt.getSuccess(), is(true)); final String logs = getAttemptLogs(client, attempt.getId()); - assertThat(logs, containsString("python python")); - assertThat(logs, containsString("python [u'python', u'-v']")); + assertThat(logs, containsString("python python3")); + assertThat(logs, containsString("python ['python3', '-v']")); } @Test diff --git a/digdag-tests/src/test/resources/acceptance/echo_params/echo_params.dig b/digdag-tests/src/test/resources/acceptance/echo_params/echo_params.dig index 342bdc32af..6ec6e82a7a 100644 --- a/digdag-tests/src/test/resources/acceptance/echo_params/echo_params.dig +++ b/digdag-tests/src/test/resources/acceptance/echo_params/echo_params.dig @@ -1,2 +1,3 @@ +foo: py>: scripts.echo_params.echo_params + python: python3 diff --git a/digdag-tests/src/test/resources/acceptance/echo_params/scripts/echo_params.py b/digdag-tests/src/test/resources/acceptance/echo_params/scripts/echo_params.py index 20d43e4aa6..f832415d26 100644 --- a/digdag-tests/src/test/resources/acceptance/echo_params/scripts/echo_params.py +++ b/digdag-tests/src/test/resources/acceptance/echo_params/scripts/echo_params.py @@ -1,4 +1,3 @@ -from __future__ import print_function import digdag def echo_params(): diff --git a/digdag-tests/src/test/resources/acceptance/emr/pi.py b/digdag-tests/src/test/resources/acceptance/emr/pi.py index 37029b7679..e646722533 100644 --- a/digdag-tests/src/test/resources/acceptance/emr/pi.py +++ b/digdag-tests/src/test/resources/acceptance/emr/pi.py @@ -1,4 +1,3 @@ -from __future__ import print_function # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with diff --git a/digdag-tests/src/test/resources/acceptance/params_visibility/dynamic_tasks_visibility.dig b/digdag-tests/src/test/resources/acceptance/params_visibility/dynamic_tasks_visibility.dig index d2945f9aaf..78eca3e63a 100644 --- a/digdag-tests/src/test/resources/acceptance/params_visibility/dynamic_tasks_visibility.dig +++ b/digdag-tests/src/test/resources/acceptance/params_visibility/dynamic_tasks_visibility.dig @@ -2,6 +2,7 @@ +initialize: +task: py>: helper.store_v + python: python3 value: initial +loop: @@ -20,6 +21,7 @@ if>: true _do: py>: helper.store_v + python: python3 value: override_${outer}_${inner} +dynamic_get_in_parallel: diff --git a/digdag-tests/src/test/resources/acceptance/params_visibility/params_visibility.dig b/digdag-tests/src/test/resources/acceptance/params_visibility/params_visibility.dig index 149ecba4f4..735cece70c 100644 --- a/digdag-tests/src/test/resources/acceptance/params_visibility/params_visibility.dig +++ b/digdag-tests/src/test/resources/acceptance/params_visibility/params_visibility.dig @@ -14,6 +14,7 @@ +stores: +store: py>: helper.store_v + python: python3 value: simple_store +simple_store: @@ -21,6 +22,7 @@ +overwrite: py>: helper.store_v + python: python3 value: store_overwrite +store_overwrite: @@ -29,6 +31,7 @@ +parallel_store_fork: +prepare: py>: helper.store_v + python: python3 value: prepare +fork: @@ -37,6 +40,7 @@ +a: +store: py>: helper.store_v + python: python3 value: a +dump: @@ -45,6 +49,7 @@ +b: +store: py>: helper.store_v + python: python3 value: b +dump: @@ -62,15 +67,18 @@ +a: py>: helper.store_v + python: python3 value: a +b: py>: helper.store_v + python: python3 value: b _after: [+a] +c: py>: helper.store_v + python: python3 value: c +dump: diff --git a/digdag-tests/src/test/resources/acceptance/py/config_python.dig b/digdag-tests/src/test/resources/acceptance/py/config_python.dig index bede81fa97..cb81ea86fd 100644 --- a/digdag-tests/src/test/resources/acceptance/py/config_python.dig +++ b/digdag-tests/src/test/resources/acceptance/py/config_python.dig @@ -2,8 +2,8 @@ +python_string_option: py>: scripts.echo_params.echo_params - python: "python" + python: "python3" +python_array_option: py>: scripts.echo_params.echo_params - python: ["python", "-v"] + python: ["python3", "-v"] diff --git a/digdag-tests/src/test/resources/acceptance/py/dup_module.dig b/digdag-tests/src/test/resources/acceptance/py/dup_module.dig index 1bd6f218ab..72a6a21bed 100644 --- a/digdag-tests/src/test/resources/acceptance/py/dup_module.dig +++ b/digdag-tests/src/test/resources/acceptance/py/dup_module.dig @@ -1,2 +1,3 @@ +task1: py>: tokenize.main + python: python3 diff --git a/digdag-tests/src/test/resources/acceptance/py/stacktrace_python.dig b/digdag-tests/src/test/resources/acceptance/py/stacktrace_python.dig index 7acb000543..b243af3d0c 100644 --- a/digdag-tests/src/test/resources/acceptance/py/stacktrace_python.dig +++ b/digdag-tests/src/test/resources/acceptance/py/stacktrace_python.dig @@ -1,5 +1,6 @@ +stacktrace_python: py>: scripts.stacktrace.run + python: python3 _error: echo>: ERROR_MESSAGE_BEGIN ${error.message} ERROR_MESSAGE_END diff --git a/digdag-tests/src/test/resources/acceptance/py/syntax_error_python.dig b/digdag-tests/src/test/resources/acceptance/py/syntax_error_python.dig index f71d1d1ca7..460df4e376 100644 --- a/digdag-tests/src/test/resources/acceptance/py/syntax_error_python.dig +++ b/digdag-tests/src/test/resources/acceptance/py/syntax_error_python.dig @@ -1,2 +1,3 @@ +task1: py>: scripts.syntax_error.func1 + python: python3 diff --git a/examples/tasks/check_task.py b/examples/tasks/check_task.py index 0cb645a170..b87aae9d57 100644 --- a/examples/tasks/check_task.py +++ b/examples/tasks/check_task.py @@ -1,4 +1,3 @@ -from __future__ import print_function import digdag def generate(): diff --git a/examples/tasks/conditions.py b/examples/tasks/conditions.py index 94099eada7..98a14ee3ec 100644 --- a/examples/tasks/conditions.py +++ b/examples/tasks/conditions.py @@ -1,4 +1,3 @@ -from __future__ import print_function import re import digdag diff --git a/examples/tasks/error_task.py b/examples/tasks/error_task.py index 87ac8dc2b6..f842e6638b 100644 --- a/examples/tasks/error_task.py +++ b/examples/tasks/error_task.py @@ -1,4 +1,3 @@ -from __future__ import print_function import digdag def fails(): diff --git a/examples/tasks/export_params.py b/examples/tasks/export_params.py index 656c930125..eab91d4303 100644 --- a/examples/tasks/export_params.py +++ b/examples/tasks/export_params.py @@ -1,4 +1,3 @@ -from __future__ import print_function import digdag def set_my_param(): diff --git a/examples/tasks/generate_subtasks.py b/examples/tasks/generate_subtasks.py index 2f5f82282d..4d98004bac 100644 --- a/examples/tasks/generate_subtasks.py +++ b/examples/tasks/generate_subtasks.py @@ -1,4 +1,3 @@ -from __future__ import print_function import digdag class ParallelProcess(object): diff --git a/examples/tasks/params.py b/examples/tasks/params.py index 3f32adb0f5..a220d035ea 100644 --- a/examples/tasks/params.py +++ b/examples/tasks/params.py @@ -1,4 +1,3 @@ -from __future__ import print_function import digdag def simple(data, number): diff --git a/examples/tasks/python_args.py b/examples/tasks/python_args.py index d0ee9a0fae..6ab9e4b8b3 100644 --- a/examples/tasks/python_args.py +++ b/examples/tasks/python_args.py @@ -1,4 +1,3 @@ -from __future__ import print_function import digdag def required_arguments(required1, required2):