diff --git a/integration-tests/source/jobs/JobsIntegrationTests.cpp b/integration-tests/source/jobs/JobsIntegrationTests.cpp index bab0bb83..82a2843e 100644 --- a/integration-tests/source/jobs/JobsIntegrationTests.cpp +++ b/integration-tests/source/jobs/JobsIntegrationTests.cpp @@ -33,6 +33,9 @@ static constexpr char VERIFY_PACKAGES_REMOVED_JOB_DOC[] = "{ \"version\": \"1.0\", \"steps\": [ { \"action\": { \"name\": \"Verify Packages Removed\", \"type\": " "\"runHandler\", \"input\": { \"handler\": \"verify-packages-removed.sh\", \"args\": [ \"dos2unix\" ], \"path\": " "\"default\" }, \"runAsUser\": \"root\" } }]}"; +static constexpr char RUN_COMMAND_PRINT_GREETING_JOB_DOC[] = + "{ \"version\": \"1.0\", \"steps\": [ { \"action\": { \"name\": \"Print Greeting\", \"type\": " + "\"runCommand\", \"input\": { \"command\": \"echo,Hello World\" }, \"runAsUser\": \"root\" } }]}"; class TestJobsFixture : public ::testing::Test { @@ -82,3 +85,11 @@ TEST_F(TestJobsFixture, RemovePackages) ASSERT_EQ(resourceHandler->GetJobExecutionStatusWithRetry(jobId), JobExecutionStatus::SUCCEEDED); } + +TEST_F(TestJobsFixture, PrintGreeting) +{ + string jobId = "Print-Greeting-" + resourceHandler->GetTimeStamp(); + resourceHandler->CreateJob(jobId, RUN_COMMAND_PRINT_GREETING_JOB_DOC); + + ASSERT_EQ(resourceHandler->GetJobExecutionStatusWithRetry(jobId), JobExecutionStatus::SUCCEEDED); +} \ No newline at end of file diff --git a/sample-job-docs/run-shellcommand.json b/sample-job-docs/run-shellcommand.json new file mode 100644 index 00000000..5d02efba --- /dev/null +++ b/sample-job-docs/run-shellcommand.json @@ -0,0 +1,16 @@ +{ + "_comment": "This sample JSON file can be used for creating a new directory newDir at the current filepath.", + "version": "1.0", + "steps": [ + { + "action": { + "name": "Create New Directory", + "type": "runCommand", + "input": { + "command": "mkdir,newDir" + }, + "runAsUser": "root" + } + } + ] +} \ No newline at end of file diff --git a/source/jobs/JobDocument.cpp b/source/jobs/JobDocument.cpp index a2cce655..e606d7d0 100644 --- a/source/jobs/JobDocument.cpp +++ b/source/jobs/JobDocument.cpp @@ -3,17 +3,22 @@ #include "JobDocument.h" #include "../logging/LoggerFactory.h" +#include "../util/StringUtils.h" #include +#include +#include using namespace std; using namespace Aws::Iot::DeviceClient::Jobs; using namespace Aws::Iot; using namespace Aws::Crt; using namespace Aws::Iot::DeviceClient::Logging; +using namespace Aws::Iot::DeviceClient::Util; constexpr char LoadableFromJobDocument::TAG[]; constexpr char PlainJobDocument::ACTION_TYPE_RUN_HANDLER[]; +constexpr char PlainJobDocument::ACTION_TYPE_RUN_COMMAND[]; constexpr char PlainJobDocument::JSON_KEY_VERSION[]; constexpr char PlainJobDocument::JSON_KEY_INCLUDESTDOUT[]; @@ -52,16 +57,16 @@ void PlainJobDocument::LoadFromJobDocument(const JsonView &json) if (json.ValueExists(jsonKey) && json.GetJsonObject(jsonKey).IsString()) { JobAction jobAction; + JobAction::ActionHandlerInput temp; // Save Job Action name and handler field value with operation field value jobAction.name = json.GetString(jsonKey).c_str(); - jobAction.input.handler = jobAction.name; + temp.handler = jobAction.name; jsonKey = JSON_KEY_ARGS; if (json.ValueExists(jsonKey) && json.GetJsonObject(jsonKey).IsListType()) { - jobAction.input.args = ParseToVectorString(json.GetJsonObject(jsonKey)); + temp.args = Util::ParseToVectorString(json.GetJsonObject(jsonKey)); } - // Old Schema only supports runHandler type of action jobAction.type = ACTION_TYPE_RUN_HANDLER; @@ -74,9 +79,11 @@ void PlainJobDocument::LoadFromJobDocument(const JsonView &json) jsonKey = JSON_KEY_PATH; if (json.ValueExists(jsonKey) && json.GetJsonObject(jsonKey).IsString()) { - jobAction.input.path = json.GetString(jsonKey).c_str(); + temp.path = json.GetString(jsonKey).c_str(); } + jobAction.handlerInput = temp; + steps.push_back(jobAction); } } @@ -127,18 +134,6 @@ void PlainJobDocument::LoadFromJobDocument(const JsonView &json) } } -vector PlainJobDocument::ParseToVectorString(const JsonView &json) -{ - vector plainVector; - - for (const auto &i : json.AsArray()) - { - // cppcheck-suppress useStlAlgorithm - plainVector.push_back(i.AsString().c_str()); - } - return plainVector; -} - bool PlainJobDocument::Validate() const { if (version.empty()) @@ -201,7 +196,7 @@ void PlainJobDocument::JobCondition::LoadFromJobDocument(const JsonView &json) jsonKey = JSON_KEY_CONDITION_VALUE; if (json.ValueExists(jsonKey) && json.GetJsonObject(jsonKey).IsListType()) { - conditionValue = ParseToVectorString(json.GetJsonObject(jsonKey)); + conditionValue = Util::ParseToVectorString(json.GetJsonObject(jsonKey)); } jsonKey = JSON_KEY_TYPE; @@ -233,6 +228,9 @@ constexpr char PlainJobDocument::JobAction::JSON_KEY_INPUT[]; constexpr char PlainJobDocument::JobAction::JSON_KEY_RUNASUSER[]; constexpr char PlainJobDocument::JobAction::JSON_KEY_ALLOWSTDERR[]; constexpr char PlainJobDocument::JobAction::JSON_KEY_IGNORESTEPFAILURE[]; +const static std::set SUPPORTED_ACTION_TYPES{ + Aws::Iot::DeviceClient::Jobs::PlainJobDocument::ACTION_TYPE_RUN_HANDLER, + Aws::Iot::DeviceClient::Jobs::PlainJobDocument::ACTION_TYPE_RUN_COMMAND}; void PlainJobDocument::JobAction::LoadFromJobDocument(const JsonView &json) { @@ -251,9 +249,18 @@ void PlainJobDocument::JobAction::LoadFromJobDocument(const JsonView &json) jsonKey = JSON_KEY_INPUT; if (json.ValueExists(jsonKey)) { - ActionInput temp; - temp.LoadFromJobDocument(json.GetJsonObject(jsonKey)); - input = temp; + if (type == PlainJobDocument::ACTION_TYPE_RUN_HANDLER) + { + ActionHandlerInput temp; + temp.LoadFromJobDocument(json.GetJsonObject(jsonKey)); + handlerInput = temp; + } + else if (type == PlainJobDocument::ACTION_TYPE_RUN_COMMAND) + { + ActionCommandInput temp; + temp.LoadFromJobDocument(json.GetJsonObject(jsonKey)); + commandInput = temp; + } } jsonKey = JSON_KEY_RUNASUSER; @@ -289,29 +296,39 @@ bool PlainJobDocument::JobAction::Validate() const return false; } - if (type != ACTION_TYPE_RUN_HANDLER) + if (SUPPORTED_ACTION_TYPES.count(type) == 0) { LOGM_ERROR( TAG, "*** %s: Required field Action Type with invalid value: %s ***", DeviceClient::Jobs::DC_INVALID_JOB_DOC, - type.c_str()); + Util::Sanitize(type).c_str()); return false; } - if (!input.Validate()) + if (type == PlainJobDocument::ACTION_TYPE_RUN_HANDLER) { - return false; + if (!handlerInput->Validate()) + { + return false; + } + } + else if (type == PlainJobDocument::ACTION_TYPE_RUN_COMMAND) + { + if (!commandInput->Validate()) + { + return false; + } } return true; } -constexpr char PlainJobDocument::JobAction::ActionInput::JSON_KEY_HANDLER[]; -constexpr char PlainJobDocument::JobAction::ActionInput::JSON_KEY_ARGS[]; -constexpr char PlainJobDocument::JobAction::ActionInput::JSON_KEY_PATH[]; +constexpr char PlainJobDocument::JobAction::ActionHandlerInput::JSON_KEY_HANDLER[]; +constexpr char PlainJobDocument::JobAction::ActionHandlerInput::JSON_KEY_ARGS[]; +constexpr char PlainJobDocument::JobAction::ActionHandlerInput::JSON_KEY_PATH[]; -void PlainJobDocument::JobAction::ActionInput::LoadFromJobDocument(const JsonView &json) +void PlainJobDocument::JobAction::ActionHandlerInput::LoadFromJobDocument(const JsonView &json) { const char *jsonKey = JSON_KEY_HANDLER; if (json.ValueExists(jsonKey) && json.GetJsonObject(jsonKey).IsString()) @@ -322,7 +339,7 @@ void PlainJobDocument::JobAction::ActionInput::LoadFromJobDocument(const JsonVie jsonKey = JSON_KEY_ARGS; if (json.ValueExists(jsonKey) && json.GetJsonObject(jsonKey).IsListType()) { - args = ParseToVectorString(json.GetJsonObject(jsonKey)); + args = Util::ParseToVectorString(json.GetJsonObject(jsonKey)); } jsonKey = JSON_KEY_PATH; @@ -332,7 +349,7 @@ void PlainJobDocument::JobAction::ActionInput::LoadFromJobDocument(const JsonVie } } -bool PlainJobDocument::JobAction::ActionInput::Validate() const +bool PlainJobDocument::JobAction::ActionHandlerInput::Validate() const { if (handler.empty()) { @@ -343,3 +360,49 @@ bool PlainJobDocument::JobAction::ActionInput::Validate() const return true; } + +constexpr char PlainJobDocument::JobAction::ActionCommandInput::JSON_KEY_COMMAND[]; + +void PlainJobDocument::JobAction::ActionCommandInput::LoadFromJobDocument(const JsonView &json) +{ + const char *jsonKey = JSON_KEY_COMMAND; + if (json.ValueExists(jsonKey)) + { + string commandString = json.GetString(jsonKey).c_str(); + + if (!commandString.empty()) + { + vector tokens = Util::SplitStringByComma(commandString); + for (auto token : tokens) + { + Util::replace_all(token, R"(\,)", ","); + // trim all leading and trailing space characters including tabs, newlines etc. + command.emplace_back(Util::TrimCopy(token, " \t\n\v\f\r")); + } + } + } +} + +bool PlainJobDocument::JobAction::ActionCommandInput::Validate() const +{ + if (command.empty()) + { + LOGM_ERROR( + TAG, "*** %s: Required field ActionInput command is missing ***", DeviceClient::Jobs::DC_INVALID_JOB_DOC); + return false; + } + + auto isSpace = [](const char &c) { return isspace(static_cast(c)); }; + const auto &firstCommand = command.front(); + if (find_if(firstCommand.cbegin(), firstCommand.cend(), isSpace) != firstCommand.cend()) + { + LOGM_ERROR( + TAG, + "*** %s: Required field ActionInput command's first word contains space characters: %s ***", + DeviceClient::Jobs::DC_INVALID_JOB_DOC, + Util::Sanitize(firstCommand).c_str()); + return false; + } + + return true; +} \ No newline at end of file diff --git a/source/jobs/JobDocument.h b/source/jobs/JobDocument.h index 1ecd5b9b..b5140418 100644 --- a/source/jobs/JobDocument.h +++ b/source/jobs/JobDocument.h @@ -35,9 +35,9 @@ namespace Aws { void LoadFromJobDocument(const JsonView &json) override; bool Validate() const override; - static std::vector ParseToVectorString(const JsonView &json); static constexpr char ACTION_TYPE_RUN_HANDLER[] = "runHandler"; + static constexpr char ACTION_TYPE_RUN_COMMAND[] = "runCommand"; static constexpr char JSON_KEY_VERSION[] = "version"; static constexpr char JSON_KEY_INCLUDESTDOUT[] = "includeStdOut"; @@ -87,7 +87,10 @@ namespace Aws std::string name; std::string type; - struct ActionInput : public LoadableFromJobDocument + /** + * ActionHandlerInput - Invokes a handler script specified in a job document. + */ + struct ActionHandlerInput : public LoadableFromJobDocument { void LoadFromJobDocument(const JsonView &json) override; bool Validate() const override; @@ -100,7 +103,21 @@ namespace Aws Optional> args; Optional path; }; - ActionInput input; + Optional handlerInput; + + /** + * ActionCommandInput - Invokes arbitrary commands specified in a job document. + */ + struct ActionCommandInput : public LoadableFromJobDocument + { + void LoadFromJobDocument(const JsonView &json) override; + bool Validate() const override; + + static constexpr char JSON_KEY_COMMAND[] = "command"; + + std::vector command; + }; + Optional commandInput; Optional runAsUser{""}; Optional allowStdErr; Optional ignoreStepFailure{false}; diff --git a/source/jobs/JobEngine.cpp b/source/jobs/JobEngine.cpp index 5c1a705d..aaae1294 100644 --- a/source/jobs/JobEngine.cpp +++ b/source/jobs/JobEngine.cpp @@ -139,10 +139,10 @@ void JobEngine::exec_action(PlainJobDocument::JobAction action, const std::strin string command; if (action.type == PlainJobDocument::ACTION_TYPE_RUN_HANDLER) { - // build command + // build command for runHandler type try { - command = buildCommand(action.input.path, action.input.handler, jobHandlerDir); + command = buildCommand(action.handlerInput->path, action.handlerInput->handler, jobHandlerDir); } catch (exception &e) { @@ -153,6 +153,11 @@ void JobEngine::exec_action(PlainJobDocument::JobAction action, const std::strin return; } } + else if (action.type == PlainJobDocument::ACTION_TYPE_RUN_COMMAND) + { + // build commands for runCommand type + command = action.commandInput->command.front(); + } else { LOG_ERROR(TAG, "Job Document received with invalid action type."); @@ -161,13 +166,22 @@ void JobEngine::exec_action(PlainJobDocument::JobAction action, const std::strin } ostringstream argsStringForLogging; - if (action.input.args.has_value()) + if (action.type == RUN_HANDLER_TYPE && action.handlerInput->args.has_value()) { - for (const auto &eachArgument : action.input.args.value()) + // build logstream for runHandler to print out on console + for (const auto &eachArgument : action.handlerInput->args.value()) { argsStringForLogging << eachArgument << " "; } } + else if (action.type == RUN_COMMAND_TYPE) + { + // build logstream for runCommand to print out on console + for (size_t i = 1; i < action.commandInput->command.size(); i++) + { + argsStringForLogging << action.commandInput->command.at(i) << " "; + } + } else { LOG_INFO( @@ -181,7 +195,15 @@ void JobEngine::exec_action(PlainJobDocument::JobAction action, const std::strin Util::Sanitize(action.runAsUser->c_str()).c_str(), Util::Sanitize(argsStringForLogging.str()).c_str()); - int actionExecutionStatus = exec_cmd(command, action); + int actionExecutionStatus; + if (action.type == RUN_HANDLER_TYPE) + { + actionExecutionStatus = exec_handlerScript(command, action); + } + else if (action.type == RUN_COMMAND_TYPE) + { + actionExecutionStatus = exec_shellCommand(action); + } if (!action.ignoreStepFailure.value()) { @@ -228,7 +250,7 @@ int JobEngine::exec_steps(PlainJobDocument jobDocument, const std::string &jobHa return executionStatus; } -int JobEngine::exec_cmd(const string &operation, PlainJobDocument::JobAction action) +int JobEngine::exec_cmd(std::unique_ptr &argv) { // Establish some file descriptors which we'll use to redirect stdout and // stderr from the child process back into our logger @@ -249,26 +271,6 @@ int JobEngine::exec_cmd(const string &operation, PlainJobDocument::JobAction act return CMD_FAILURE; } - /** - * \brief Create char array argv[] storing arguments to pass to execvp() function. - * argv[0] executable path - * argv[1] Linux user name - * argv[2:] arguments required for executing the executable file.. - */ - size_t argSize = 0; - if (action.input.args.has_value()) - { - argSize = action.input.args->size(); - } - std::unique_ptr argv(new const char *[argSize + 3]); - argv[0] = operation.c_str(); - argv[1] = action.runAsUser->c_str(); - argv[argSize + 2] = nullptr; - for (size_t i = 0; i < argSize; i++) - { - argv[i + 2] = action.input.args->at(i).c_str(); - } - int execResult; int returnCode; int pid = vfork(); @@ -303,12 +305,14 @@ int JobEngine::exec_cmd(const string &operation, PlainJobDocument::JobAction act LOG_DEBUG(TAG, "Child process about to call execvp"); - if (execvp(operation.c_str(), const_cast(argv.get())) == -1) + auto rc = execvp(argv[0], const_cast(argv.get())); + if (rc == -1) { - LOGM_DEBUG(TAG, "Failed to invoke execvp system call to execute action step: %s ", strerror(errno)); + auto err = errno; + LOGM_ERROR(TAG, "Failed to invoke execvp system call to execute action step: %s (%d)", strerror(err), err); + _exit(rc); } - // If the exec fails we need to exit the child process - _exit(1); + _exit(0); } else { @@ -330,16 +334,169 @@ int JobEngine::exec_cmd(const string &operation, PlainJobDocument::JobAction act int waitReturn = waitpid(pid, &execResult, 0); if (waitReturn == -1) { - LOG_WARN(TAG, "Failed to wait for child process"); + LOGM_WARN(TAG, "Failed to wait for child process: %d", pid); } - LOGM_DEBUG(TAG, "JobEngine finished waiting for child process, returning %d", execResult); - returnCode = execResult; + returnCode = WEXITSTATUS(execResult); + LOGM_DEBUG(TAG, "JobEngine finished waiting for child process, returning %d", returnCode); } while (!WIFEXITED(execResult) && !WIFSIGNALED(execResult)); } return returnCode; } +int JobEngine::exec_process(std::unique_ptr &argv) +{ + int status = 0; + int execStatus = 0; + int pid = vfork(); + + if (pid < 0) + { + auto err = errno; + LOGM_ERROR(TAG, "Failed to create child process, fork returned: %s (%d)", strerror(err), err); + return CMD_FAILURE; + } + else if (pid == 0) + { + LOG_DEBUG(TAG, "Child process now running."); + + auto rc = execvp(argv[0], const_cast(argv.get())); + if (rc == -1) + { + auto err = errno; + LOGM_ERROR(TAG, "Failed to invoke execvp system call to execute action step: %s (%d)", strerror(err), err); + _exit(rc); + } + _exit(0); + } + else + { + LOGM_DEBUG(TAG, "Parent process now running, child PID is %d", pid); + do + { + // TODO: do not wait for infinite time for child process to complete + int waitReturn = waitpid(pid, &status, 0); + if (waitReturn == -1) + { + LOGM_WARN(TAG, "Failed to wait for child process: %d", pid); + } + execStatus = WEXITSTATUS(status); + LOGM_DEBUG(TAG, "JobEngine finished waiting for child process, returning %d", execStatus); + + } while (!WIFEXITED(status) && !WIFSIGNALED(status)); + } + return execStatus; +} + +int JobEngine::exec_handlerScript(const std::string &command, PlainJobDocument::JobAction action) +{ + /** + * \brief Create char array argv[] storing arguments to pass to execvp() function. + * argv[0] executable path + * argv[1] Linux user name + * argv[2:] arguments required for executing the executable file.. + */ + int actionExecutionStatus; + size_t argSize = 0; + if (action.handlerInput->args.has_value()) + { + argSize = action.handlerInput->args->size(); + } + std::unique_ptr argv(new const char *[argSize + 3]); + argv[0] = command.c_str(); + argv[1] = action.runAsUser->c_str(); + argv[argSize + 2] = nullptr; + for (size_t i = 0; i < argSize; i++) + { + argv[i + 2] = action.handlerInput->args->at(i).c_str(); + } + actionExecutionStatus = exec_cmd(argv); + return actionExecutionStatus; +} + +bool JobEngine::verifySudoAndUser(PlainJobDocument::JobAction action) +{ + int execStatus1; + // first to run command id $user and /bin/bash -c "command -v sudo" to verify user and sudo + std::unique_ptr argv1(new const char *[3]); + argv1[0] = "id"; + argv1[1] = action.runAsUser->c_str(); + argv1[2] = nullptr; + + execStatus1 = exec_process(argv1); + + if (execStatus1 == 0) + { + std::unique_ptr argv2(new const char *[4]); + argv2[0] = "/bin/bash"; + argv2[1] = "-c"; + argv2[2] = "command -v sudo"; + argv2[3] = nullptr; + + int execStatus2 = exec_process(argv2); + if (execStatus2 != 0) + { + return false; + } + } + else + { + return false; + } + return true; +} + +int JobEngine::exec_shellCommand(PlainJobDocument::JobAction action) +{ + int returnCode; + bool verification; + + verification = verifySudoAndUser(action); + + if (!verification) + { + // if one of two verification fails, execute command without "sudo" and "$user" + LOG_WARN(TAG, "username or sudo command not found"); + + size_t argSize = action.commandInput->command.size(); + std::unique_ptr argv(new const char *[argSize + 1]); + argv[argSize] = nullptr; + for (size_t i = 0; i < argSize; i++) + { + argv[i] = action.commandInput->command.at(i).c_str(); + } + // print out argv for debug + for (size_t i = 0; i < argSize; ++i) + { + LOGM_DEBUG(TAG, "argv[%lu]: %s", i, (argv.get())[i]); + } + returnCode = exec_cmd(argv); + } + else + { + // if two verifications succeeds, build command using sudo -u $user -n $@ and execute + size_t argSize = action.commandInput->command.size(); + std::unique_ptr argv(new const char *[argSize + 5]); + argv[0] = "sudo"; + argv[1] = "-u"; + argv[2] = action.runAsUser->c_str(); + argv[3] = "-n"; + argv[argSize + 4] = nullptr; + for (size_t i = 0; i < argSize; i++) + { + argv[i + 4] = action.commandInput->command.at(i).c_str(); + } + // print out argv to debug + for (size_t i = 0; i < argSize + 4; ++i) + { + LOGM_DEBUG(TAG, "argv[%lu]: %s", i, (argv.get())[i]); + } + + returnCode = exec_cmd(argv); + } + return returnCode; +} + string JobEngine::getReason(int statusCode) { ostringstream reason; diff --git a/source/jobs/JobEngine.h b/source/jobs/JobEngine.h index 4e87c90c..23ea8172 100644 --- a/source/jobs/JobEngine.h +++ b/source/jobs/JobEngine.h @@ -49,6 +49,9 @@ namespace Aws */ const char *DEFAULT_PATH_KEYWORD = "default"; + const char *RUN_HANDLER_TYPE = "runHandler"; + const char *RUN_COMMAND_TYPE = "runCommand"; + /** * \brief The number of lines received on STDERR from the child process * @@ -84,13 +87,44 @@ namespace Aws const std::string &jobHandlerDir) const; /** - * \brief Executes the given command (action) and passes the provided vector of arguments to that - * command - * @param action the command to execute - * @param args the arguments to pass to that command + * \brief Executes the argv, consists of command and arguments, using execvp(). + * This function also opens two pipes to process outputs from child processes. + * @param argv the arguments to pass to execvp() to execute + * @return an integer representing the return code of the executed process + */ + int exec_cmd(std::unique_ptr &argv); + + /** + * \brief Executes the argv, consists of command and arguments, using execvp() + * This function only returns the exit code of child processes + * @param argv the arguments to pass to execvp() to execute + * @return an integer representing the return code of the executed process + */ + int exec_process(std::unique_ptr &argv); + + /** + * \brief Verifies if "sudo" and "$user" exists + * @param action the action provided in job document to execute + * @return an boolean indicating verification succeeds or fails + */ + bool verifySudoAndUser(PlainJobDocument::JobAction action); + + /** + * \brief Builds argv for "runHandler" type of jobs and makes calls to exec_cmd() + * to execute + * @param command the command built from handler and path and used by execvp() to execute + * @param action the action provided in job document to execute * @return an integer representing the return code of the executed process */ - int exec_cmd(const std::string &operation, PlainJobDocument::JobAction action); + int exec_handlerScript(const std::string &command, PlainJobDocument::JobAction action); + + /** + * \brief Builds argv for "runCommand" type of jobs and makes calls to exec_cmd() + * to execute + * @param action the action provided in job document to execute + * @return an boolean indicating verification succeeds or fails + */ + int exec_shellCommand(PlainJobDocument::JobAction action); /** * \brief Executes the given set of steps (actions) in sequence as provided in the job document diff --git a/source/jobs/README.md b/source/jobs/README.md index 3a9f468f..33c41a09 100644 --- a/source/jobs/README.md +++ b/source/jobs/README.md @@ -64,7 +64,7 @@ Existing [Sample Job Handlers](../../sample-job-handlers) "name":"Install wget package on the device" ... ``` - `type` *string* (Required): This attribute defines the type of step to be executed. We currently only support actions of the type `runHandler` in `version` `"1.0"` of the Job Document Schema. + `type` *string* (Required): This attribute defines the type of step to be executed. We currently support actions of the type `runHandler` or `runCommand` in `version` `"1.0"` of the Job Document Schema. `runCommand` is only supported using NEW job document schema. ``` ... @@ -91,7 +91,7 @@ Existing [Sample Job Handlers](../../sample-job-handlers) `input` *JSON* (Required): This attribute defines the supporting parameters / arguments required to execute your step as part of the Job execution. - The `input` attribute further consists of three fields: `handler`, `args`, and `path`. + The `input` attribute consists of different fields between types. For `runHandler` type, it further consists of three fields: `handler`, `args`, and `path`. `handler` *string* (Required): This field declares the name of the handler script to be executed for your step as part of the Job execution. @@ -125,7 +125,19 @@ Else you can also specify a different handler directory for your step in the fol "path": "/home/ubuntu/my-job-handler-folder" ... ``` - +For `runCommand` type, `input` field consists of only one field: `command`. + +`command` *string* (Required): This field stores one command provided by the customer to be executed on devices. The format of `command` needs to be comma separated. For example, `aws iot describe-endpoint --endpoint-type XXX --region XXX --endpoint https://xxxxx` needs to be comma separated into `"aws,iot,describe-endpoint,--endpoint-type,XXX,--region,XXX,--endpoint,https://xxxxx"`. If command itself contains comma, the comma needs to be escaped. For example, `echo Hello, I am Device Client.` needs to be transformed into `echo,Hello\\, I am Device Client.`. The first string of the command cannot contain any space characters. + +Lastly, the permission used to execute `command` will be the value of `runAsUser`. if no user is provided, it will use the permission of the user which runs device client. Therefore, if a command needs to use `root` permission, simply provide `root` for `runAsUser` instead of adding `sudo` in front of `command`, because device client will execute the command in the form of `sudo -u $user -n command` if `sudo` is installed and `user` is found. +``` +... +"command": "echo,Hello World!" +... +``` + +**Note**: Once a job document is received by Device Client and parsed successfully, `input` will be stored into `handlerInput` or `commandInput` according to the `type` of the job document. `handlerInput` consists of `handler`, `args` and `path` and `commandInput` only consists of `command`. + **Example of Step Field:** ``` @@ -151,6 +163,24 @@ Else you can also specify a different handler directory for your step in the fol ... ``` + ``` + ... + "steps": [ + { + "action": { + "name": "print-greeting", + "type": "runCommand", + "runAsUser": "root", + "ignoreStepFailure":"true", + "input": { + "command": "echo,Hello World!" + } + } + } + ] + ... + ``` + **Example of Final Step Field:** ``` diff --git a/source/util/StringUtils.cpp b/source/util/StringUtils.cpp index 3d07b2e9..102f603c 100644 --- a/source/util/StringUtils.cpp +++ b/source/util/StringUtils.cpp @@ -5,6 +5,7 @@ #include "../config/Config.h" #include #include +#include using namespace std; @@ -97,13 +98,45 @@ namespace Aws string TrimRightCopy(string s, const string &any) { return s.erase(s.find_last_not_of(any) + 1); } - // cppcheck-suppress unusedFunction string TrimCopy(string s, const string &any) { s.erase(0, s.find_first_not_of(any)); s.erase(s.find_last_not_of(any) + 1); return s; } + + vector ParseToVectorString(const JsonView &json) + { + vector plainVector; + + for (const auto &i : json.AsArray()) + { + // cppcheck-suppress useStlAlgorithm + plainVector.push_back(i.AsString().c_str()); + } + return plainVector; + } + + vector SplitStringByComma(const string &stringToSplit) + { + regex delim{R"((\\,|[^,])+)"}; + + vector tokens; + copy( + sregex_token_iterator{begin(stringToSplit), end(stringToSplit), delim}, + sregex_token_iterator{}, + back_inserter(tokens)); + return tokens; + } + + void replace_all(string &inout, const string &what, const string &with) + { + for (string::size_type pos{}; inout.npos != (pos = inout.find(what.data(), pos, what.length())); + pos += with.length()) + { + inout.replace(pos, what.length(), with.data(), with.length()); + } + } } // namespace Util } // namespace DeviceClient } // namespace Iot diff --git a/source/util/StringUtils.h b/source/util/StringUtils.h index 5bd10d40..4fb403da 100644 --- a/source/util/StringUtils.h +++ b/source/util/StringUtils.h @@ -1,9 +1,12 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +#include #include #include +using namespace Aws::Crt; + namespace Aws { namespace Iot @@ -82,6 +85,29 @@ namespace Aws * @return string with leftmost and rightmost characters removed */ std::string TrimCopy(std::string s, const std::string &any); + + /** + * \brief Given an input JsonView object, will return vector of string form of JsonView + * @param json Input JsonView + * @return vector of string form of JsonView + */ + std::vector ParseToVectorString(const JsonView &json); + + /** + * \brief Split the given input string by comma into a vector of string, escaped comma will be ignored + * @param stringToSplit Input string + * @return vector of string split by comma + */ + std::vector SplitStringByComma(const std::string &stringToSplit); + + /** + * \brief Replace all substrings found inside the given input string + * @param inout String to be manipulated + * @param what Substring within @param inout to be replaced + * @param with Input string that replaces @param what + * @return String with all selected substring replaced + */ + void replace_all(std::string &inout, const std::string &what, const std::string &with); } // namespace Util } // namespace DeviceClient } // namespace Iot diff --git a/test/jobs/TestJobDocument.cpp b/test/jobs/TestJobDocument.cpp index ac664804..674fbc20 100644 --- a/test/jobs/TestJobDocument.cpp +++ b/test/jobs/TestJobDocument.cpp @@ -36,13 +36,20 @@ void AssertConditionEqual( } } -void AssertInputEqual( - const PlainJobDocument::JobAction::ActionInput &input1, - const PlainJobDocument::JobAction::ActionInput &input2) +void AssertHandlerInputEqual( + const Optional &input1, + const Optional &input2) { - ASSERT_STREQ(input1.handler.c_str(), input2.handler.c_str()); - AssertVectorEqual(input1.args, input2.args); - ASSERT_STREQ(input1.path->c_str(), input2.path->c_str()); + ASSERT_STREQ(input1->handler.c_str(), input2->handler.c_str()); + AssertVectorEqual(input1->args, input2->args); + ASSERT_STREQ(input1->path->c_str(), input2->path->c_str()); +} + +void AssertCommandInputEqual( + const Optional &input1, + const Optional &input2) +{ + AssertVectorEqual(input1->command, input2->command); } void AssertStepEqual(const vector &step1, const vector &step2) @@ -54,7 +61,14 @@ void AssertStepEqual(const vector &step1, const vec { ASSERT_STREQ(iterator1->name.c_str(), iterator2->name.c_str()); ASSERT_STREQ(iterator1->type.c_str(), iterator2->type.c_str()); - AssertInputEqual(iterator1->input, iterator2->input); + if (iterator1->type == "runHandler") + { + AssertHandlerInputEqual(iterator1->handlerInput, iterator2->handlerInput); + } + else + { + AssertCommandInputEqual(iterator1->commandInput, iterator2->commandInput); + } ASSERT_STREQ(iterator1->runAsUser->c_str(), iterator2->runAsUser->c_str()); ASSERT_TRUE(iterator1->allowStdErr.value() == iterator2->allowStdErr.value()); ASSERT_TRUE(iterator2->ignoreStepFailure); @@ -108,6 +122,18 @@ TEST(JobDocument, SampleJobDocument) "ignoreStepFailure": "true" } }, + { + "action": { + "name": "displayDirectory", + "type": "runCommand", + "input": { + "command": "ls,/tmp" + }, + "runAsUser": "user1", + "allowStdErr": 8, + "ignoreStepFailure": "true" + } + }, { "action": { "name": "validateAppStatus", @@ -154,9 +180,19 @@ TEST(JobDocument, SampleJobDocument) for (const auto &i : jobDocument.steps) { cout << i.name.c_str() << "\n"; - for (const auto &j : *i.input.args) + if (i.type == "runHandler") { - cout << j.c_str() << "\n"; + for (const auto &j : *i.handlerInput->args) + { + cout << j.c_str() << "\n"; + } + } + else + { + for (const auto &j : i.commandInput->command) + { + cout << j.c_str() << "\n"; + } } } @@ -187,14 +223,17 @@ TEST(JobDocument, SampleJobDocument) cout << i.type->c_str() << "\n"; } - PlainJobDocument::JobAction action1, action2, action3; + PlainJobDocument::JobAction action1, action2, action3, action4; + PlainJobDocument::JobAction::ActionHandlerInput handlerInput1, handlerInput2, handlerInput4; + PlainJobDocument::JobAction::ActionCommandInput commandInput3; std::vector steps; action1.name = "downloadJobHandler"; action1.type = "runHandler"; - action1.input.handler = "download-file.sh"; - action1.input.args = {{"presignedUrl"}, {"/tmp/aws-iot-device-client/"}}; - action1.input.path = "path to handler"; + handlerInput1.handler = "download-file.sh"; + handlerInput1.args = {{"presignedUrl"}, {"/tmp/aws-iot-device-client/"}}; + handlerInput1.path = "path to handler"; + action1.handlerInput = handlerInput1; action1.runAsUser = "user1"; action1.allowStdErr = 8; action1.ignoreStepFailure = true; @@ -202,39 +241,52 @@ TEST(JobDocument, SampleJobDocument) action2.name = "installApplicationAndReboot"; action2.type = "runHandler"; - action2.input.handler = "install-app.sh"; - action2.input.args = {{"applicationName"}, {"active"}}; - action2.input.path = "path to handler"; + handlerInput2.handler = "install-app.sh"; + handlerInput2.args = {{"applicationName"}, {"active"}}; + handlerInput2.path = "path to handler"; + action2.handlerInput = handlerInput2; action2.runAsUser = "user1"; action2.allowStdErr = 8; action2.ignoreStepFailure = true; steps.push_back(action2); - action3.name = "validateAppStatus"; - action3.type = "runHandler"; - action3.input.handler = "validate-app-status.sh"; - action3.input.args = {{"applicationName"}, {"active"}}; - action3.input.path = "path to handler"; + action3.name = "displayDirectory"; + action3.type = "runCommand"; + commandInput3.command = {{"ls"}, {"/tmp"}}; + action3.commandInput = commandInput3; action3.runAsUser = "user1"; action3.allowStdErr = 8; action3.ignoreStepFailure = true; steps.push_back(action3); + action4.name = "validateAppStatus"; + action4.type = "runHandler"; + handlerInput4.handler = "validate-app-status.sh"; + handlerInput4.args = {{"applicationName"}, {"active"}}; + handlerInput4.path = "path to handler"; + action4.handlerInput = handlerInput4; + action4.runAsUser = "user1"; + action4.allowStdErr = 8; + action4.ignoreStepFailure = true; + steps.push_back(action4); + AssertStepEqual(steps, jobDocument.steps); PlainJobDocument::JobAction finalAction; + PlainJobDocument::JobAction::ActionHandlerInput finalhandlerInput; finalAction.name = "deleteDownloadedHandler"; finalAction.type = "runHandler"; - finalAction.input.handler = "validate-app-status.sh"; - finalAction.input.args = {{"applicationName"}, {"active"}}; - finalAction.input.path = "path to handler"; + finalhandlerInput.handler = "validate-app-status.sh"; + finalhandlerInput.args = {{"applicationName"}, {"active"}}; + finalhandlerInput.path = "path to handler"; + finalAction.handlerInput = finalhandlerInput; finalAction.runAsUser = "user1"; finalAction.allowStdErr = 8; finalAction.ignoreStepFailure = true; ASSERT_STREQ(finalAction.name.c_str(), jobDocument.finalStep->name.c_str()); ASSERT_STREQ(finalAction.type.c_str(), jobDocument.finalStep->type.c_str()); - AssertInputEqual(finalAction.input, jobDocument.finalStep->input); + AssertHandlerInputEqual(finalAction.handlerInput, jobDocument.finalStep->handlerInput); ASSERT_STREQ(finalAction.runAsUser->c_str(), jobDocument.finalStep->runAsUser->c_str()); ASSERT_EQ(finalAction.allowStdErr.value(), jobDocument.finalStep->allowStdErr.value()); ASSERT_TRUE(jobDocument.finalStep->ignoreStepFailure); @@ -356,6 +408,15 @@ TEST(JobDocument, MinimumJobDocument) } } }, + { + "action": { + "name": "displayDirectory", + "type": "runCommand", + "input": { + "command": "ls,/tmp" + } + } + }, { "action": { "name": "validateAppStatus", @@ -468,4 +529,103 @@ TEST(JobDocument, MissingRequiredFieldsValue) jobDocument.LoadFromJobDocument(jsonView); ASSERT_FALSE(jobDocument.Validate()); +} + +TEST(JobDocument, CommandFieldsIsEmpty) +{ + constexpr char jsonString[] = R"( +{ + "version": "1.0", + "steps": [ + { + "action": { + "name": "displayDirectory", + "type": "runCommand", + "input": { + "command": + } + } + } + ] +})"; + + JsonObject jsonObject(jsonString); + JsonView jsonView = jsonObject.View(); + + PlainJobDocument jobDocument; + jobDocument.LoadFromJobDocument(jsonView); + + ASSERT_FALSE(jobDocument.Validate()); +} + +TEST(JobDocument, CommandContainsSpaceCharacters) +{ + constexpr char jsonString[] = R"( +{ + "version": "1.0", + "steps": [ + { + "action": { + "name": "displayDirectory", + "type": "runCommand", + "input": { + "command": " \n echo \t, Hello World " + } + } + } + ] +})"; + + JsonObject jsonObject(jsonString); + JsonView jsonView = jsonObject.View(); + + PlainJobDocument jobDocument; + jobDocument.LoadFromJobDocument(jsonView); + + ASSERT_TRUE(jobDocument.Validate()); +} + +TEST(JobDocument, SpaceCharactersContainedWithinFirstWordOfCommand) +{ + constexpr char jsonString[] = R"( +{ + "version": "1.0", + "steps": [ + { + "action": { + "name": "displayDirectory", + "type": "runCommand", + "input": { + "command": " aws iot \t,describe-endpoint" + } + } + } + ] +})"; + + JsonObject jsonObject(jsonString); + JsonView jsonView = jsonObject.View(); + + PlainJobDocument jobDocument; + jobDocument.LoadFromJobDocument(jsonView); + + ASSERT_FALSE(jobDocument.Validate()); +} + +TEST(JobDocument, oldJobDocumentCompatibility) +{ + constexpr char jsonString[] = R"( +{ + "operation": "download-file.sh", + "args": ["https://github.com/awslabs/aws-iot-device-client/archive/refs/tags/v1.3.tar.gz", "/tmp/Downloaded_File.tar.gz"], + "path": "default" +})"; + + JsonObject jsonObject(jsonString); + JsonView jsonView = jsonObject.View(); + + PlainJobDocument jobDocument; + jobDocument.LoadFromJobDocument(jsonView); + + ASSERT_TRUE(jobDocument.Validate()); } \ No newline at end of file diff --git a/test/jobs/TestJobEngine.cpp b/test/jobs/TestJobEngine.cpp index de19aaad..9c2b51c5 100644 --- a/test/jobs/TestJobEngine.cpp +++ b/test/jobs/TestJobEngine.cpp @@ -18,19 +18,34 @@ PlainJobDocument::JobAction createJobAction( string type, string handler, std::vector args, + std::vector command, string path, + const char *runAsUser, bool ignoreStepFailure) { - PlainJobDocument::JobAction::ActionInput input; - input.handler = handler; - input.args = args; - input.path = path; - PlainJobDocument::JobAction action; action.name = name; action.type = type; + if (runAsUser != nullptr) + { + action.runAsUser = runAsUser; + } action.ignoreStepFailure = ignoreStepFailure; - action.input = input; + + if (type == "runHandler") + { + PlainJobDocument::JobAction::ActionHandlerInput input; + input.handler = handler; + input.args = args; + input.path = path; + action.handlerInput = input; + } + else + { + PlainJobDocument::JobAction::ActionCommandInput input; + input.command = command; + action.commandInput = input; + } return action; } @@ -58,6 +73,7 @@ PlainJobDocument createTestJobDocument( const string testHandlerDirectoryPath = "/tmp/device-client-tests"; const string successHandlerPath = testHandlerDirectoryPath + "/successHandler"; const string errorHandlerPath = testHandlerDirectoryPath + "/errorHandler"; +const string successCreatedFile = testHandlerDirectoryPath + "/test-success"; const string testStdout = "This is test stdout"; const string testStderr = "This is test stderr"; @@ -84,6 +100,7 @@ class TestJobEngine : public testing::Test { std::remove(successHandlerPath.c_str()); std::remove(errorHandlerPath.c_str()); + std::remove(successCreatedFile.c_str()); std::remove(testHandlerDirectoryPath.c_str()); } }; @@ -92,8 +109,9 @@ TEST_F(TestJobEngine, ExecuteStepsHappy) { vector steps; vector args; - steps.push_back( - createJobAction("testAction", "runHandler", "successHandler", args, "/tmp/device-client-tests/", false)); + vector command; + steps.push_back(createJobAction( + "testAction", "runHandler", "successHandler", args, command, "/tmp/device-client-tests/", nullptr, false)); PlainJobDocument jobDocument = createTestJobDocument(steps, true); JobEngine jobEngine; @@ -106,10 +124,11 @@ TEST_F(TestJobEngine, ExecuteSucceedThenFail) { vector steps; vector args; - steps.push_back( - createJobAction("testAction", "runHandler", "successHandler", args, "/tmp/device-client-tests/", false)); - PlainJobDocument::JobAction finalStep = - createJobAction("testAction", "runHandler", "errorHandler", args, "/tmp/device-client-tests/", false); + vector command; + steps.push_back(createJobAction( + "testAction", "runHandler", "successHandler", args, command, "/tmp/device-client-tests/", nullptr, false)); + PlainJobDocument::JobAction finalStep = createJobAction( + "testAction", "runHandler", "errorHandler", args, command, "/tmp/device-client-tests/", nullptr, false); PlainJobDocument jobDocument = createTestJobDocument(steps, finalStep, true); JobEngine jobEngine; @@ -123,8 +142,9 @@ TEST_F(TestJobEngine, ExecuteFinalStepOnly) { vector steps; vector args; - PlainJobDocument::JobAction finalStep = - createJobAction("testAction", "runHandler", "successHandler", args, "/tmp/device-client-tests/", false); + vector command; + PlainJobDocument::JobAction finalStep = createJobAction( + "testAction", "runHandler", "successHandler", args, command, "/tmp/device-client-tests/", nullptr, false); PlainJobDocument jobDocument = createTestJobDocument(steps, finalStep, true); JobEngine jobEngine; @@ -137,8 +157,9 @@ TEST_F(TestJobEngine, ExecuteStepsError) { vector steps; vector args; - steps.push_back( - createJobAction("testAction", "runHandler", "errorHandler", args, "/tmp/device-client-tests/", false)); + vector command; + steps.push_back(createJobAction( + "testAction", "runHandler", "errorHandler", args, command, "/tmp/device-client-tests/", nullptr, false)); PlainJobDocument jobDocument = createTestJobDocument(steps, true); JobEngine jobEngine; @@ -158,4 +179,52 @@ TEST_F(TestJobEngine, ExecuteNoSteps) ASSERT_EQ(executionStatus, 0); ASSERT_EQ(jobEngine.getStdOut().length(), 0); ASSERT_EQ(jobEngine.getStdErr().length(), 0); +} + +TEST_F(TestJobEngine, ExecuteRunCommandWithInvalidUser) +{ + vector steps; + vector args; + vector command; + command.emplace_back("touch"); + command.emplace_back(successCreatedFile); + steps.push_back(createJobAction("testCreateFile", "runCommand", "", args, command, "", "fake", false)); + PlainJobDocument jobDocument = createTestJobDocument(steps, true); + JobEngine jobEngine; + + int executionStatus = jobEngine.exec_steps(jobDocument, testHandlerDirectoryPath); + ASSERT_EQ(executionStatus, 0); + ASSERT_TRUE(FileUtils::FileExists(successCreatedFile)); +} + +TEST_F(TestJobEngine, ExecuteRunCommandWithEmptyUser) +{ + vector steps; + vector args; + vector command; + command.emplace_back("touch"); + command.emplace_back(successCreatedFile); + steps.push_back(createJobAction("testCreateFile", "runCommand", "", args, command, "", nullptr, false)); + PlainJobDocument jobDocument = createTestJobDocument(steps, true); + JobEngine jobEngine; + + int executionStatus = jobEngine.exec_steps(jobDocument, testHandlerDirectoryPath); + ASSERT_EQ(executionStatus, 0); + ASSERT_TRUE(FileUtils::FileExists(successCreatedFile)); +} + +TEST_F(TestJobEngine, ExecuteRunCommandWithUser) +{ + vector steps; + vector args; + vector command; + command.emplace_back("touch"); + command.emplace_back(successCreatedFile); + steps.push_back(createJobAction("testCreateFile", "runCommand", "", args, command, "", "root", false)); + PlainJobDocument jobDocument = createTestJobDocument(steps, true); + JobEngine jobEngine; + + int executionStatus = jobEngine.exec_steps(jobDocument, testHandlerDirectoryPath); + ASSERT_EQ(executionStatus, 0); + ASSERT_TRUE(FileUtils::FileExists(successCreatedFile)); } \ No newline at end of file diff --git a/test/util/TestStringUtils.cpp b/test/util/TestStringUtils.cpp index 0bb0b414..90aea86f 100644 --- a/test/util/TestStringUtils.cpp +++ b/test/util/TestStringUtils.cpp @@ -4,6 +4,7 @@ #include "../../source/config/Config.h" #include "../../source/util/StringUtils.h" #include "gtest/gtest.h" +#include using namespace std; using namespace Aws::Iot::DeviceClient; @@ -121,4 +122,39 @@ TEST(StringUtils, trimMultiChar) ASSERT_EQ("b", TrimCopy("/a/b/c/", "/ac")); // Match. ASSERT_EQ("/a/b/c/", TrimCopy("/a/b/c/", "ac")); // No match. ASSERT_EQ("", TrimCopy("", "/")); // Empty string. +} + +TEST(StringUtils, ParseToStringVector) +{ + constexpr char jsonString[] = R"( +{ + "args": ["hello", "world"] +})"; + JsonObject jsonObject(jsonString); + JsonView jsonView = jsonObject.View(); + + vector expected{"hello", "world"}; + vector actual = ParseToVectorString(jsonView.GetJsonObject("args")); + + ASSERT_TRUE(expected.size() == actual.size()); + ASSERT_TRUE(equal(expected.begin(), expected.end(), actual.begin())); +} + +TEST(StringUtils, SplitStringByComma) +{ + string stringToSplit{"hello,world\\,!"}; + vector expected{"hello", "world\\,!"}; + vector actual = SplitStringByComma(stringToSplit); + + ASSERT_TRUE(expected.size() == actual.size()); + ASSERT_TRUE(equal(expected.begin(), expected.end(), actual.begin())); +} + +TEST(StringUtils, replace_all) +{ + string actual{"hello\\,world!"}; + string expected{"hello,world!"}; + replace_all(actual, R"(\,)", ","); + + ASSERT_STREQ(actual.c_str(), expected.c_str()); } \ No newline at end of file