From 3443e093b38139c50d4f7d9733f4b260baef6deb Mon Sep 17 00:00:00 2001 From: Tri Hoang Vo Date: Fri, 29 May 2020 13:04:37 +0200 Subject: [PATCH 1/7] Sandbox ansible execution in a docker container * this commit makes sure ansible and scripts are executed from inside a sandbox container in the following steps: * We first generate all configuration files for the execution in the ansible RecipePath on the host machine (i.e., ansible.cfg, hosts, run.ansible.yml, wrapper, .vault_pass). Then start the sandbox container and bind mount the ansibleRecipePath in the sandbox (at /home/ansible). * bind mount the ssh agent socket on the host machine to the sandbox (at /home/ssh-agent), and pass the path to the socket (i.e., SSH_AUTH_SOCK) as an environment variable to the sandbox container. * bind mount the overlay, which contains deployment artifacts in the sandbox (at /home/overlay). * after starting the sandbox, reuse the CMD implementation to execute ansible inside the sandbox container. Ansible then writes output logs back to the file system (in /home/ansible/*-out.csv) so that the output handler of the CMD can read the logs. * add container configs for security hardening the sandbox (e.g., specify a non-root user to run the container, do not allow new privileges escalation, remove ALL capabilities, add yorc config to limit cpu to 0.5 and memory to 256m to avoid DoS). * if OpenSSH is enabled, specify the location where ansible can write ControlPath in the container (at /home/ansible/cp) to avoid ansible to have a write permission denied. * add config to specify the location where ansible can write factCaches in the container (at /home/ansible/facts_cache). Each task should have its own facts_cache so that one task cannot overwrite caches from the other parallel ones unexpectedly. --- config/config.go | 12 +++ config/config_test.go | 2 +- pkg/ansible/Dockerfile | 15 ++++ prov/ansible/consul_test.go | 3 + prov/ansible/execution.go | 128 ++++++++++++++++++------------ prov/ansible/execution_ansible.go | 48 +++++++---- prov/ansible/execution_scripts.go | 50 +++++++----- prov/ansible/execution_test.go | 13 +++ prov/ansible/sandbox.go | 117 +++++++++++++++++++++++---- prov/ansible/sandbox_test.go | 41 +++++++--- 10 files changed, 313 insertions(+), 116 deletions(-) create mode 100644 pkg/ansible/Dockerfile diff --git a/config/config.go b/config/config.go index e79bc2872..dccdb049b 100644 --- a/config/config.go +++ b/config/config.go @@ -84,6 +84,15 @@ const DefaultUpgradesConcurrencyLimit = 1000 // DefaultSSHConnectionTimeout is the default timeout for SSH connections const DefaultSSHConnectionTimeout = 10 * time.Second +// DefaultSandboxWorkDir is the default workdir in the sandbox container where we mount the ansible recipes +const DefaultSandboxWorkDir = "/home/ansible" + +// DefaultSandboxOverlayDir is the default directory in the sandbox container where we mount the overlay +const DefaultSandboxOverlayDir = "/home/overlay" + +// DefaultSandboxMountAgentSocket is the default location in the sandbox container where we mount the ssh agent socket +const DefaultSandboxMountAgentSocket = "/ssh-agent" + // Configuration holds config information filled by Cobra and Viper (see commands package for more information) type Configuration struct { Ansible Ansible `yaml:"ansible,omitempty" mapstructure:"ansible"` @@ -120,6 +129,9 @@ type DockerSandbox struct { Command []string `mapstructure:"command"` Entrypoint []string `mapstructure:"entrypoint"` Env []string `mapstructure:"env"` + User string `mapstructure:"user"` + Cpus string `mapstructure:"cpus"` + Memory string `mapstructure:"memory"` } // HostedOperations holds the configuration for operations executed on the orechestrator host (eg. with an operation_host equals to ORECHESTRATOR) diff --git a/config/config_test.go b/config/config_test.go index 7a4a09841..3809d93d3 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -210,7 +210,7 @@ func TestHostedOperations_Format(t *testing.T) { }{ {"DefaultValues", fields{}, `{UnsandboxedOperationsAllowed:false DefaultSandbox:}`}, {"AllowUnsandboxed", fields{UnsandboxedOperationsAllowed: true}, `{UnsandboxedOperationsAllowed:true DefaultSandbox:}`}, - {"DefaultSandboxConfigured", fields{DefaultSandbox: &DockerSandbox{Image: "alpine:3.7", Command: []string{"cmd", "arg"}}}, `{UnsandboxedOperationsAllowed:false DefaultSandbox:&{Image:alpine:3.7 Command:[cmd arg] Entrypoint:[] Env:[]}}`}, + {"DefaultSandboxConfigured", fields{DefaultSandbox: &DockerSandbox{Image: "alpine:3.7", Command: []string{"cmd", "arg"}, User: "1000:1000", Cpus: "1", Memory: "200m"}}, `{UnsandboxedOperationsAllowed:false DefaultSandbox:&{Image:alpine:3.7 Command:[cmd arg] Entrypoint:[] Env:[] User:1000:1000 Cpus:1 Memory:200m}}`}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/ansible/Dockerfile b/pkg/ansible/Dockerfile new file mode 100644 index 000000000..ebba9b564 --- /dev/null +++ b/pkg/ansible/Dockerfile @@ -0,0 +1,15 @@ +# Dockerfile for building the sandbox container + +FROM alpine:3.10 + +ENV ANSIBLE_VERSION=2.9.9 + +RUN apk add --no-cache --progress bash python3 openssl ca-certificates openssh sshpass && \ + addgroup -g 1000 ansible && \ + adduser -D -s /bin/bash -g ansible -G ansible -u 1000 ansible && \ + if [ ! -e /usr/bin/python ]; then ln -sf /usr/bin/python3 /usr/bin/python; fi && \ + apk --update add --virtual build-dependencies python3-dev libffi-dev openssl-dev build-base && \ + pip3 install --upgrade pip && \ + pip3 install ansible==${ANSIBLE_VERSION} && \ + apk del build-dependencies && \ + rm -rf /var/cache/apk/* \ No newline at end of file diff --git a/prov/ansible/consul_test.go b/prov/ansible/consul_test.go index ab4b15fdf..38bc38a19 100644 --- a/prov/ansible/consul_test.go +++ b/prov/ansible/consul_test.go @@ -43,4 +43,7 @@ func TestRunConsulAnsiblePackageTests(t *testing.T) { t.Run("TestLogAnsibleOutputInConsulFromScriptFailure", func(t *testing.T) { testLogAnsibleOutputInConsulFromScriptFailure(t) }) + t.Run("TestSandbox", func(t *testing.T) { + testCreatesandbox(t) + }) } diff --git a/prov/ansible/execution.go b/prov/ansible/execution.go index c42756ae9..cbb6225eb 100644 --- a/prov/ansible/execution.go +++ b/prov/ansible/execution.go @@ -61,6 +61,7 @@ print(os.environ['VAULT_PASSWORD']) ` const ansibleConfigDefaultsHeader = "defaults" +const ansibleConfigSSHConnection = "ssh_connection" const ansibleInventoryHostsHeader = "target_hosts" const ansibleInventoryHostedHeader = "hosted_operations" const ansibleInventoryHostsVarsHeader = ansibleInventoryHostsHeader + ":vars" @@ -73,6 +74,7 @@ var ansibleDefaultConfig = map[string]map[string]string{ "stdout_callback": "yaml", "nocows": "1", }, + ansibleConfigSSHConnection: map[string]string{}, } var ansibleFactCaching = map[string]string{ @@ -139,7 +141,7 @@ type execution interface { } type ansibleRunner interface { - runAnsible(ctx context.Context, retry bool, currentInstance, ansibleRecipePath string) error + generateRunAnsible(ctx context.Context, currentInstance, ansibleRecipePath string) (handler outputHandler, err error) } type executionCommon struct { @@ -788,26 +790,13 @@ func (e *executionCommon) execute(ctx context.Context, retry bool) error { } func (e *executionCommon) generateHostConnectionForOrchestratorOperation(ctx context.Context, buffer *bytes.Buffer) error { - if e.cli != nil && e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { - var err error - e.containerID, err = createSandbox(ctx, e.cli, e.cfg.Ansible.HostedOperations.DefaultSandbox, e.deploymentID) - if err != nil { - return err - } - buffer.WriteString(" ansible_connection=docker ansible_host=") - buffer.WriteString(e.containerID) - } else if e.cfg.Ansible.HostedOperations.UnsandboxedOperationsAllowed { - buffer.WriteString(" ansible_connection=local") - } else { - actualRootCause := "there is no sandbox configured to handle it" - if e.cli == nil { - actualRootCause = "connection to docker failed (see logs)" - } - - err := errors.Errorf("Ansible provisioning: you are trying to execute an operation on the orchestrator host but %s and execution on the actual orchestrator host is disallowed by configuration", actualRootCause) + if e.cfg.Ansible.HostedOperations.DefaultSandbox == nil && !e.cfg.Ansible.HostedOperations.UnsandboxedOperationsAllowed { + err := errors.Errorf("Ansible provisioning: you are trying to execute an operation on the orchestrator host " + + "but there is no sandbox configured to handle it and execution on the actual orchestrator host is disallowed by configuration") events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).Registerf("%v", err) return err } + buffer.WriteString(" ansible_connection=local") return nil } @@ -1087,11 +1076,13 @@ func (e *executionCommon) executeWithCurrentInstance(ctx context.Context, retry tarPath := filepath.Join(ansibleRecipePath, artifactName+".tar") buildArchive(e.OverlayPath, artifactPath, tarPath) } - - err = e.ansibleRunner.runAnsible(ctx, retry, currentInstance, ansibleRecipePath) + outputHandler, err := e.ansibleRunner.generateRunAnsible(ctx, currentInstance, ansibleRecipePath) if err != nil { return err } + if err = e.executePlaybook(ctx, retry, ansibleRecipePath, outputHandler); err != nil { + return err + } if e.HaveOutput { outputsFiles, err := filepath.Glob(filepath.Join(ansibleRecipePath, "*-out.csv")) if err != nil { @@ -1208,9 +1199,51 @@ func (e *executionCommon) getInstanceIDFromHost(host string) (string, error) { func (e *executionCommon) executePlaybook(ctx context.Context, retry bool, ansibleRecipePath string, handler outputHandler) error { - cmd := executil.Command(ctx, "ansible-playbook", "-i", "hosts", "run.ansible.yml", "--vault-password-file", filepath.Join(ansibleRecipePath, ".vault_pass")) env := os.Environ() env = append(env, "VAULT_PASSWORD="+e.vaultToken) + var sshAgentSocket string + if !e.cfg.DisableSSHAgent { + // Check if SSHAgent is needed + sshAgent, err := e.configureSSHAgent(ctx) + if err != nil { + return errors.Wrap(err, "failed to configure SSH agent for ansible-playbook execution") + } + if sshAgent != nil { + sshAgentSocket = sshAgent.Socket + log.Debugf("Add SSH_AUTH_SOCK env var for ssh-agent") + if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { + env = append(env, "SSH_AUTH_SOCK="+config.DefaultSandboxMountAgentSocket) + } else { + env = append(env, "SSH_AUTH_SOCK="+sshAgentSocket) + } + defer func() { + err = sshAgent.RemoveAllKeys() + if err != nil { + log.Debugf("Warning: failed to remove all SSH agents keys due to error:%+v", err) + } + err = sshAgent.Stop() + if err != nil { + log.Debugf("Warning: failed to stop SSH agent due to error:%+v", err) + } + }() + } + } + var cmd *executil.Cmd + if e.cli != nil && e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { + log.Debugf("Start sandbox container with mount directories ansibleRecipePath: %s, overlayPath: %s", ansibleRecipePath, e.OverlayPath) + var err error + if e.containerID, err = createSandbox(ctx, e.cli, e.cfg.Ansible.HostedOperations.DefaultSandbox, e.deploymentID, + ansibleRecipePath, e.OverlayPath, sshAgentSocket, env); err != nil { + return err + } + log.Debugf("Execute ansible-playbook in sandbox container: %s", e.containerID) + cmd = executil.Command(ctx, "docker", "exec", "--workdir", config.DefaultSandboxWorkDir, e.containerID, "ansible-playbook", "-i", "hosts", "run.ansible.yml", "--vault-password-file", ".vault_pass") + } else { + log.Debugf("Execute ansible-playbook on localhost") + cmd = executil.Command(ctx, "ansible-playbook", "-i", "hosts", "run.ansible.yml", "--vault-password-file", filepath.Join(ansibleRecipePath, ".vault_pass")) + cmd.Env = env + } + if _, err := os.Stat(filepath.Join(ansibleRecipePath, "run.ansible.retry")); retry && (err == nil || !os.IsNotExist(err)) { cmd.Args = append(cmd.Args, "--limit", filepath.Join("@", ansibleRecipePath, "run.ansible.retry")) } @@ -1228,31 +1261,8 @@ func (e *executionCommon) executePlaybook(ctx context.Context, retry bool, } else { cmd.Args = append(cmd.Args, "-c", "paramiko") } - - if !e.cfg.DisableSSHAgent { - // Check if SSHAgent is needed - sshAgent, err := e.configureSSHAgent(ctx) - if err != nil { - return errors.Wrap(err, "failed to configure SSH agent for ansible-playbook execution") - } - if sshAgent != nil { - log.Debugf("Add SSH_AUTH_SOCK env var for ssh-agent") - env = append(env, "SSH_AUTH_SOCK="+sshAgent.Socket) - defer func() { - err = sshAgent.RemoveAllKeys() - if err != nil { - log.Debugf("Warning: failed to remove all SSH agents keys due to error:%+v", err) - } - err = sshAgent.Stop() - if err != nil { - log.Debugf("Warning: failed to stop SSH agent due to error:%+v", err) - } - }() - } - } } cmd.Dir = ansibleRecipePath - cmd.Env = env errbuf := events.NewBufferedLogEntryWriter() cmd.Stderr = errbuf @@ -1400,12 +1410,32 @@ func (e *executionCommon) generateAnsibleConfigurationFile( ansibleConfig := getAnsibleConfigFromDefault() - // Adding settings whose values are known at runtime, related to the deployment - // directory path - ansibleConfig[ansibleConfigDefaultsHeader]["retry_files_save_path"] = ansibleRecipePath - if e.CacheFacts { - ansibleFactCaching["fact_caching_connection"] = path.Join(ansiblePath, "facts_cache") + // Adding settings whose values are known at runtime, related to the deployment directory path + if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { + ansibleConfig[ansibleConfigDefaultsHeader]["retry_files_save_path"] = config.DefaultSandboxWorkDir + // Specify where ansible can write its temp files in the sandbox container. + // (By default, ansible writes to ~/.ansible/tmp on host machine and may cause permission denied from inside the container) + ansibleConfig[ansibleConfigDefaultsHeader]["local_tmp"] = path.Join(config.DefaultSandboxWorkDir, "tmp") + // Specify where ansible writes SSH control path sockets in the sandbox container for SSH multiplexing. + // (By default, ansible writes to ~/.ansible/cp on host machine and may cause permission denied from inside the container) + // Each sandbox contrainer should have its own control path + if e.cfg.Ansible.UseOpenSSH { + ansibleConfig[ansibleConfigSSHConnection]["control_path_dir"] = path.Join(config.DefaultSandboxWorkDir, "cp") + } + } else { + ansibleConfig[ansibleConfigDefaultsHeader]["retry_files_save_path"] = ansibleRecipePath + } + if e.CacheFacts { + if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { + // location in sandbox to save ansible gathering facts + // each yorc task should have its own facts (i.e., do not share the gathering facts from different tasks + // even in the same deploymentID) so that one task do not overwrite the facts of the other ones unexpectedly + // (e.g., if share facts, set "become: true" in one task overwrites the USER fact from ubuntu to root). + ansibleFactCaching["fact_caching_connection"] = path.Join(config.DefaultSandboxWorkDir, "facts_cache") + } else { + ansibleFactCaching["fact_caching_connection"] = path.Join(ansibleRecipePath, "facts_cache") + } for k, v := range ansibleFactCaching { ansibleConfig[ansibleConfigDefaultsHeader][k] = v } diff --git a/prov/ansible/execution_ansible.go b/prov/ansible/execution_ansible.go index 41ff2b0be..ef662f305 100644 --- a/prov/ansible/execution_ansible.go +++ b/prov/ansible/execution_ansible.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "fmt" + "github.com/ystia/yorc/v4/config" "io/ioutil" "os" "path/filepath" @@ -65,10 +66,24 @@ type executionAnsible struct { isAlienAnsible bool } -func (e *executionAnsible) runAnsible(ctx context.Context, retry bool, currentInstance, ansibleRecipePath string) error { +func (e *executionAnsible) generateRunAnsible(ctx context.Context, currentInstance, ansibleRecipePath string) (outputHandler, error) { var err error + outputHandler := &playbookOutputHandler{} + // for operation on host machine, set the ansible destination folder to the ansible recipe path on host machine + // for operation on sandbox, set the ansible destination folder and overlay to the default mount path inside the container + var overlayPathOnHost, destFolder string + if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { + overlayPathOnHost = e.OverlayPath + e.OverlayPath = config.DefaultSandboxOverlayDir + destFolder = config.DefaultSandboxWorkDir + defer func() { + e.OverlayPath = overlayPathOnHost + }() + } else { + destFolder = ansibleRecipePath + } if !e.isAlienAnsible { - e.PlaybookPath, err = filepath.Abs(filepath.Join(e.OverlayPath, e.Primary)) + e.PlaybookPath = filepath.Join(e.OverlayPath, e.Primary) } else { var playbook string for _, envInput := range e.EnvInputs { @@ -81,19 +96,19 @@ func (e *executionAnsible) runAnsible(ctx context.Context, retry bool, currentIn err = errors.New("No PLAYBOOK_ENTRY input found for an alien4cloud ansible implementation") } if err == nil { - e.PlaybookPath, err = filepath.Abs(filepath.Join(e.OverlayPath, filepath.Dir(e.Primary), playbook)) + e.PlaybookPath = filepath.Join(e.OverlayPath, filepath.Dir(e.Primary), playbook) } } if err != nil { events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) - return err + return outputHandler, err } ansibleGroupsVarsPath := filepath.Join(ansibleRecipePath, "group_vars") if err = os.MkdirAll(ansibleGroupsVarsPath, 0775); err != nil { err = errors.Wrap(err, "Failed to create group_vars directory: ") events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) - return err + return outputHandler, err } var buffer bytes.Buffer for _, envInput := range e.EnvInputs { @@ -103,7 +118,7 @@ func (e *executionAnsible) runAnsible(ctx context.Context, retry bool, currentIn } v, err := e.encodeEnvInputValue(envInput, ansibleRecipePath) if err != nil { - return err + return outputHandler, err } buffer.WriteString(fmt.Sprintf("%s: %s", envInput.Name, v)) buffer.WriteString("\n") @@ -135,19 +150,19 @@ func (e *executionAnsible) runAnsible(ctx context.Context, retry bool, currentIn for contextKey, contextValue := range e.CapabilitiesCtx { v, err := e.encodeTOSCAValue(contextValue, ansibleRecipePath) if err != nil { - return err + return outputHandler, err } buffer.WriteString(fmt.Sprintf("%s: %s", contextKey, v)) buffer.WriteString("\n") } buffer.WriteString("dest_folder: \"") - buffer.WriteString(ansibleRecipePath) + buffer.WriteString(destFolder) buffer.WriteString("\"\n") if err = ioutil.WriteFile(filepath.Join(ansibleGroupsVarsPath, "all.yml"), buffer.Bytes(), 0664); err != nil { err = errors.Wrap(err, "Failed to write global group vars file: ") events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) - return err + return outputHandler, err } if e.HaveOutput { @@ -162,7 +177,7 @@ func (e *executionAnsible) runAnsible(ctx context.Context, retry bool, currentIn if err = ioutil.WriteFile(filepath.Join(ansibleRecipePath, "outputs.csv.j2"), buffer.Bytes(), 0664); err != nil { err = errors.Wrap(err, "Failed to generate operation outputs file: ") events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) - return err + return outputHandler, err } } @@ -184,21 +199,22 @@ func (e *executionAnsible) runAnsible(ctx context.Context, retry bool, currentIn if err != nil { err = errors.Wrap(err, "Failed to generate ansible playbook") events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) - return err + return outputHandler, err } if err = tmpl.Execute(&buffer, e); err != nil { err = errors.Wrap(err, "Failed to Generate ansible playbook template") events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) - return err + return outputHandler, err } if err = ioutil.WriteFile(filepath.Join(ansibleRecipePath, "run.ansible.yml"), buffer.Bytes(), 0664); err != nil { err = errors.Wrap(err, "Failed to write playbook file") events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) - return err + return outputHandler, err + } + if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { + e.OverlayPath = overlayPathOnHost } - events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelDEBUG, e.deploymentID).RegisterAsString(fmt.Sprintf("Ansible recipe for node %q: executing %q on remote host(s)", e.NodeName, filepath.Base(e.PlaybookPath))) - outputHandler := &playbookOutputHandler{execution: e, context: ctx} - return e.executePlaybook(ctx, retry, ansibleRecipePath, outputHandler) + return &playbookOutputHandler{execution: e, context: ctx}, err } diff --git a/prov/ansible/execution_scripts.go b/prov/ansible/execution_scripts.go index eecbe8748..5ec30b4fb 100644 --- a/prov/ansible/execution_scripts.go +++ b/prov/ansible/execution_scripts.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "fmt" + "github.com/ystia/yorc/v4/config" "io/ioutil" "path/filepath" "strings" @@ -207,27 +208,32 @@ func getExecutionScriptTemplateFnMap(e *executionCommon, ansibleRecipePath strin } } -func (e *executionScript) runAnsible(ctx context.Context, retry bool, currentInstance, ansibleRecipePath string) error { +func (e *executionScript) generateRunAnsible(ctx context.Context, currentInstance, ansibleRecipePath string) (outputHandler, error) { var err error - e.ScriptToRun, err = filepath.Abs(filepath.Join(e.OverlayPath, e.Primary)) - if err != nil { - return errors.Wrap(err, "Failed to retrieve script absolute path") - } - - e.DestFolder, err = filepath.Abs(ansibleRecipePath) - if err != nil { - return errors.Wrap(err, "Failed to retrieve script wrapper absolute path") + outputHandler := &scriptOutputHandler{} + // for operation on host machine, set the ansible destination folder to the ansible recipe path on host machine + // for operation on sandbox, set the ansible destination folder and overlay to the default mount path inside the container + overlayPathOnHost := e.OverlayPath + wrapperLocationOnHost := filepath.Join(ansibleRecipePath, "wrapper") + if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { + e.OverlayPath = config.DefaultSandboxOverlayDir + e.DestFolder = config.DefaultSandboxWorkDir + defer func() { + e.OverlayPath = overlayPathOnHost + }() + } else { + e.DestFolder = ansibleRecipePath } - + e.ScriptToRun = filepath.Join(e.OverlayPath, e.Primary) e.WrapperLocation = filepath.Join(e.DestFolder, "wrapper") - outputHandler := &scriptOutputHandler{execution: e, context: ctx, instanceName: currentInstance} - + scriptHandler := scriptOutputHandler{execution: e, context: ctx, instanceName: currentInstance} + outputHandler = &scriptHandler var buffer bytes.Buffer tmpl := template.New("execTemplate") tmpl = tmpl.Delims("[[[", "]]]") - tmpl = tmpl.Funcs(getExecutionScriptTemplateFnMap(e.executionCommon, ansibleRecipePath, outputHandler.getWrappedCommand)) + tmpl = tmpl.Funcs(getExecutionScriptTemplateFnMap(e.executionCommon, ansibleRecipePath, scriptHandler.getWrappedCommand)) wrapTemplate := template.New("execTemplate") wrapTemplate = wrapTemplate.Delims("[[[", "]]]") if e.isPython { @@ -236,17 +242,17 @@ func (e *executionScript) runAnsible(ctx context.Context, retry bool, currentIns wrapTemplate, err = tmpl.Parse(scriptCustomWrapper) } if err != nil { - return err + return outputHandler, err } if err := wrapTemplate.Execute(&buffer, e); err != nil { err = errors.Wrap(err, "Failed to Generate wrapper template") events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) - return err + return outputHandler, err } - if err := ioutil.WriteFile(e.WrapperLocation, buffer.Bytes(), 0664); err != nil { + if err := ioutil.WriteFile(wrapperLocationOnHost, buffer.Bytes(), 0664); err != nil { err = errors.Wrap(err, "Failed to write playbook file") events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) - return err + return outputHandler, err } buffer.Reset() @@ -254,24 +260,24 @@ func (e *executionScript) runAnsible(ctx context.Context, retry bool, currentIns if err != nil { err = errors.Wrap(err, "Failed to Generate ansible playbook") events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) - return err + return outputHandler, err } if err = tmpl.Execute(&buffer, e); err != nil { err = errors.Wrap(err, "Failed to Generate ansible playbook template") events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) - return err + return outputHandler, err } if err = ioutil.WriteFile(filepath.Join(ansibleRecipePath, "run.ansible.yml"), buffer.Bytes(), 0664); err != nil { err = errors.Wrap(err, "Failed to write playbook file") events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) - return err + return outputHandler, err } scriptPath, err := filepath.Abs(filepath.Join(e.OverlayPath, e.Primary)) if err != nil { - return err + return outputHandler, err } events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelDEBUG, e.deploymentID).RegisterAsString(fmt.Sprintf("Ansible recipe for node %q: executing %q on remote host(s)", e.NodeName, filepath.Base(scriptPath))) - return e.executePlaybook(ctx, retry, ansibleRecipePath, outputHandler) + return outputHandler, err } diff --git a/prov/ansible/execution_test.go b/prov/ansible/execution_test.go index 4a270fd74..57f6e9719 100644 --- a/prov/ansible/execution_test.go +++ b/prov/ansible/execution_test.go @@ -744,6 +744,19 @@ func testExecutionGenerateAnsibleConfig(t *testing.T) { len(resultMap[ansibleConfigDefaultsHeader]), "Missing entries in ansible config file with fact caching, content: %q", content) + // Test enabling ansible sandbox, OpenSSH also adds ssh_connection control path + execution.cfg.Ansible.UseOpenSSH = true + execution.cfg.Ansible.HostedOperations.DefaultSandbox = &config.DockerSandbox{} + err = execution.generateAnsibleConfigurationFile("ansiblePath", yorcConfig.WorkingDirectory) + require.NoError(t, err, "Error generating ansible config file") + resultMap, content = readAnsibleConfigSettings(t, cfgPath) + assert.Equal(t, + initialConfigMapLength+len(ansibleFactCaching)+2, + len(resultMap[ansibleConfigDefaultsHeader]), + "Missing entries in ansible config file with local_tmp, content: %q", content) + assert.Equal(t, 1, len(resultMap[ansibleConfigSSHConnection]), + "Missing entries in ansible config file with ssh connection, content: %q", content) + // Test with ansible config settings in Yorc server configuration // one of them overriding Yorc default ansible config setting newSettingName := "special_context_filesystems" diff --git a/prov/ansible/sandbox.go b/prov/ansible/sandbox.go index ada656976..8b41db8b0 100644 --- a/prov/ansible/sandbox.go +++ b/prov/ansible/sandbox.go @@ -16,6 +16,8 @@ package ansible import ( "context" + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/opts" "io/ioutil" "time" @@ -31,7 +33,11 @@ import ( "github.com/ystia/yorc/v4/log" ) -func createSandbox(ctx context.Context, cli *client.Client, sandboxCfg *config.DockerSandbox, deploymentID string) (string, error) { +var nanoCPUs opts.NanoCPUs +var memoryInBytes opts.MemBytes + +func createSandbox(ctx context.Context, cli *client.Client, sandboxCfg *config.DockerSandbox, deploymentID, + ansibleRecipePath, overlayPath, sshAgentSocket string, env []string) (string, error) { // check context is cancelable if ctx.Done() == nil { @@ -42,25 +48,28 @@ func createSandbox(ctx context.Context, cli *client.Client, sandboxCfg *config.D if sandboxCfg.Image == "" { return "", errors.New("Docker sandbox for orchestrator-hosted operation misconfigured, image option is missing") } - - events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelDEBUG, deploymentID).Registerf("Pulling docker image: %s", sandboxCfg.Image) - pullResp, err := cli.ImagePull(ctx, sandboxCfg.Image, types.ImagePullOptions{}) - if pullResp != nil { - b, errRead := ioutil.ReadAll(pullResp) - if errRead == nil && len(b) > 0 { - events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelDEBUG, deploymentID).Registerf("Pulled docker image: %s", string(b)) - } - pullResp.Close() - } + // pull docker image if not exists + _, _, err := cli.ImageInspectWithRaw(ctx, sandboxCfg.Image) if err != nil { - return "", errors.Wrapf(err, "Failed to pull docker image %q", sandboxCfg.Image) + events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelDEBUG, deploymentID).Registerf("Docker image %s not exists. "+ + "Pulling docker image.", sandboxCfg.Image) + pullResp, err := cli.ImagePull(ctx, sandboxCfg.Image, types.ImagePullOptions{}) + if pullResp != nil { + b, errRead := ioutil.ReadAll(pullResp) + if errRead == nil && len(b) > 0 { + events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelDEBUG, deploymentID).Registerf("Pulled docker image: %s", string(b)) + } + pullResp.Close() + } + if err != nil { + return "", errors.Wrapf(err, "Failed to pull docker image %q", sandboxCfg.Image) + } } - + env = append(env, sandboxCfg.Env...) cc := &container.Config{ Image: sandboxCfg.Image, - Env: sandboxCfg.Env, + Env: env, } - if len(sandboxCfg.Command) == 0 && len(sandboxCfg.Entrypoint) == 0 { cc.Entrypoint = strslice.StrSlice{"python"} cc.Cmd = strslice.StrSlice{"-c", "import time;time.sleep(31536000);"} @@ -71,9 +80,53 @@ func createSandbox(ctx context.Context, cli *client.Client, sandboxCfg *config.D if len(sandboxCfg.Entrypoint) > 0 { cc.Entrypoint = strslice.StrSlice(sandboxCfg.Entrypoint) } - + // run sandbox container with non-root user + if sandboxCfg.User != "" { + cc.User = sandboxCfg.User + } + // limit resources for sandbox container to avoid DoS attacks + c, err := getNanoCPUs(sandboxCfg.Cpus) + if err != nil { + return "", err + } + m, err := getMemoryInBytes(sandboxCfg.Memory) + if err != nil { + return "", err + } hc := &container.HostConfig{ AutoRemove: true, + // Security hardening for the sandbox container + // do not allow privilege escalation + SecurityOpt: []string{"no-new-privileges=true"}, + // run the sandbox container with read-only root file system + ReadonlyRootfs: true, + // drop all capabilities + CapDrop: []string{"ALL"}, + Resources: container.Resources{ + NanoCPUs: c, + Memory: m, + }, + // mount volumes + Mounts: []mount.Mount{ + { + Type: "bind", + Source: ansibleRecipePath, + Target: config.DefaultSandboxWorkDir, + ReadOnly: false, + }, + { + Type: "bind", + Source: overlayPath, + Target: config.DefaultSandboxOverlayDir, + ReadOnly: true, + }, + { + Type: "bind", + Source: sshAgentSocket, + Target: config.DefaultSandboxMountAgentSocket, + ReadOnly: true, + }, + }, } events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelDEBUG, deploymentID).Registerf("Creating docker container from image: %s", sandboxCfg.Image) @@ -108,3 +161,35 @@ func stopSandboxOnContextCancellation(ctx context.Context, cli *client.Client, d } events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelDEBUG, deploymentID).Registerf("Docker container with id %q removed", containerID) } + +// getNanoCPUs converts user defined cpus string to nano cpus presentation +// defaults cpu to 0.5 cpu if not set +func getNanoCPUs(c string) (int64, error) { + if nanoCPUs != 0 { + return nanoCPUs.Value(), nil + } + if c != "" { + if err := nanoCPUs.Set(c); err != nil { + return 0, err + } + } else { + _ = nanoCPUs.Set("0.5") + } + return nanoCPUs.Value(), nil +} + +// getMemoryInBytes converts user defined memory string to memory in bytes presentation +// defaults memory to 256m if not set +func getMemoryInBytes(m string) (int64, error) { + if memoryInBytes != 0 { + return memoryInBytes.Value(), nil + } + if m != "" { + if err := memoryInBytes.Set(m); err != nil { + return 0, err + } + } else { + _ = memoryInBytes.Set("256m") + } + return memoryInBytes.Value(), nil +} diff --git a/prov/ansible/sandbox_test.go b/prov/ansible/sandbox_test.go index 50eefa2cf..d1bb96ac2 100644 --- a/prov/ansible/sandbox_test.go +++ b/prov/ansible/sandbox_test.go @@ -18,6 +18,7 @@ import ( "bytes" "context" "encoding/json" + "github.com/docker/docker/api/types" "io/ioutil" "net/http" "strings" @@ -62,44 +63,60 @@ func newFailingDockerMockForPaths(t *testing.T, paths []string, status int) *cli }) } -func newJSONBodyResponseDockerMock(t *testing.T, p string, obj interface{}) *client.Client { +func newJSONBodyResponseDockerMock(t *testing.T, p map[string]interface{}) *client.Client { return newMockDockerClient(t, func(r *http.Request) (*http.Response, error) { if r == nil { return nil, errors.New("nil http request") } - resp := &http.Response{StatusCode: http.StatusOK} - b, err := json.Marshal(obj) - if err != nil { - return nil, err - } - if strings.Contains(r.URL.Path, p) { - resp.Body = ioutil.NopCloser(bytes.NewReader(b)) + resp := &http.Response{} + for k, v := range p { + if strings.Contains(r.URL.Path, k) { + resp.StatusCode = http.StatusOK + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + resp.Body = ioutil.NopCloser(bytes.NewReader(b)) + break + } } return resp, nil }) } -func Test_createSandbox(t *testing.T) { +func testCreatesandbox(t *testing.T) { type args struct { cli *client.Client sandboxCfg *config.DockerSandbox deploymentID string } + // mock success request and response + request := map[string]interface{}{ + "/images/busybox:latest": &types.ImageInspect{ + ID: "exist_image_id", + }, + "/containers/create": &container.ContainerCreateCreatedBody{ + ID: "myid", + }, + "/containers/myid/start": []byte(""), + "/containers/myid/stop": []byte(""), + "/containers/myid": []byte(""), + } tests := []struct { name string args args want string wantErr bool }{ - {"FailOnPull", args{newFailingDockerMockForPaths(t, []string{"/images/create"}, 404), &config.DockerSandbox{Image: "busybox:latest"}, "d1"}, "", true}, - {"Success", args{newJSONBodyResponseDockerMock(t, "/containers/create", &container.ContainerCreateCreatedBody{ID: "myid"}), &config.DockerSandbox{Image: "busybox:latest"}, "d1"}, "myid", false}, + {"FailOnPull", args{newFailingDockerMockForPaths(t, []string{"/images/create", "/images/busybox:latest"}, 404), &config.DockerSandbox{Image: "busybox:latest"}, "d1"}, "", true}, + {"Success", args{newJSONBodyResponseDockerMock(t, request), &config.DockerSandbox{Image: "busybox:latest"}, "d1"}, "myid", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctx, cf := context.WithCancel(context.Background()) defer cf() - got, err := createSandbox(ctx, tt.args.cli, tt.args.sandboxCfg, tt.args.deploymentID) + got, err := createSandbox(ctx, tt.args.cli, tt.args.sandboxCfg, tt.args.deploymentID, "/path/host", "/path/host", "/path/host", []string{}) if (err != nil) != tt.wantErr { t.Errorf("createSandbox() error = %v, wantErr %v", err, tt.wantErr) return From 9db3089edafe65e57b14e8aef2f090e493b2a733 Mon Sep 17 00:00:00 2001 From: Tri Vo Date: Sun, 7 Jun 2020 11:13:18 +0200 Subject: [PATCH 2/7] Add config bind-address for ssh-agent, add home env for sandbox * Add bind-address for ssh-agent so that ssh-agent stores the agent sockets in the work directoy (work/ssh-agent) instead of the default /tmp folder. This allows the sandbox container to bind mount volume from inside the work dir but not the /tmp folder. * Add home environment for the sandbox container so that ansible resolves its default path "~/.ansible" to the workdir inside the sandbox correctly. Otherwise ansible resolves to /home/ on the host machine, where yorc is running. --- go.mod | 2 ++ helper/sshutil/sshutil.go | 14 ++++++++++++-- helper/sshutil/sshutil_test.go | 23 ++++++++++++++++++++++- prov/ansible/execution.go | 14 +++++++++----- prov/ansible/sandbox.go | 19 +++++++++++++------ prov/terraform/executor.go | 2 +- 6 files changed, 59 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 1b7cfd811..72b26b9dc 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/armon/go-metrics v0.3.0 github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 // indirect github.com/blang/semver v3.5.1+incompatible + github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 github.com/bradleyjkemp/cupaloy v2.3.0+incompatible // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect @@ -109,6 +110,7 @@ require ( golang.org/x/net v0.0.0-20200202094626-16171245cfb2 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e + golang.org/x/tools v0.0.0-20200302225559-9b52d559c609 gopkg.in/AlecAivazis/survey.v1 v1.6.3 gopkg.in/cookieo9/resources-go.v2 v2.0.0-20150225115733-d27c04069d0d gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect diff --git a/helper/sshutil/sshutil.go b/helper/sshutil/sshutil.go index 00dc97e13..e1a8774c3 100644 --- a/helper/sshutil/sshutil.go +++ b/helper/sshutil/sshutil.go @@ -17,6 +17,7 @@ package sshutil import ( "bytes" "fmt" + uuid "github.com/satori/go.uuid" "io" "io/ioutil" "net" @@ -268,14 +269,23 @@ func (client *SSHClient) CopyFile(source io.Reader, remotePath string, permissio return nil } -// NewSSHAgent allows to return a new SSH Agent -func NewSSHAgent(ctx context.Context) (*SSHAgent, error) { +// NewSSHAgent allows to return a new SSH Agent. If socketDir is specified, create the agent socket at /socketDir//agent +func NewSSHAgent(ctx context.Context, socketDir string) (*SSHAgent, error) { bin, err := exec.LookPath("ssh-agent") if err != nil { return nil, errors.Wrap(err, "could not find ssh-agent") } cmd := executil.Command(ctx, bin) + if socketDir != "" { + tempDir := filepath.Join(socketDir, uuid.NewV4().String()) + if err = os.MkdirAll(tempDir, 0770); err != nil { + return nil, errors.Wrapf(err, "could not create socket directory for ssh-agent at %q", tempDir) + } + bindAddress := filepath.Join(tempDir, "agent") + log.Debugf("Set ssh-agent bind-address to %q", bindAddress) + cmd.Args = append(cmd.Args, "-a", bindAddress) + } out, err := cmd.Output() if err != nil { return nil, errors.Wrap(err, "failed to run ssh-agent") diff --git a/helper/sshutil/sshutil_test.go b/helper/sshutil/sshutil_test.go index e4d0089ef..b0a64f8cd 100644 --- a/helper/sshutil/sshutil_test.go +++ b/helper/sshutil/sshutil_test.go @@ -20,6 +20,8 @@ import ( "crypto/rsa" "crypto/x509" "encoding/pem" + "github.com/stretchr/testify/assert" + "os" "testing" "github.com/stretchr/testify/require" @@ -39,7 +41,7 @@ func TestSSHAgent(t *testing.T) { privateKeyContent := string(bArray) // Create new ssh-agent - sshAg, err := NewSSHAgent(context.Background()) + sshAg, err := NewSSHAgent(context.Background(), "") require.Nil(t, err, "unexpected error while creating SSH-agent") defer func() { err = sshAg.Stop() @@ -67,6 +69,25 @@ func TestSSHAgent(t *testing.T) { keys, err = sshAg.agent.List() require.Nil(t, err) require.Len(t, keys, 0, "no key expected") + + // test create more than one agents in a specific socket directory + sshAg1, err := NewSSHAgent(context.Background(), "/tmp/ssh-agent") + require.Nil(t, err) + sshAg2, err := NewSSHAgent(context.Background(), "/tmp/ssh-agent") + require.Nil(t, err) + require.DirExists(t, "/tmp/ssh-agent") + assert.NotEqual(t, sshAg1.Socket, sshAg2.Socket) + // stop one ssh-agent does not remove the parent socket directory but the given agent socket only + err = sshAg1.Stop() + require.Nil(t, err) + require.DirExists(t, "/tmp/ssh-agent", "stop one ssh-agent removed the parent socket directory") + if _, err := os.Stat(sshAg1.Socket); err == nil { + t.Errorf("failed to remove agent socket when stopping ssh-agent %v", err) + } + require.FileExists(t, sshAg2.Socket, "stop one ssh-agent but removed another agent socket") + err = sshAg2.Stop() + err = os.RemoveAll("/tmp/ssh-agent") + require.Nil(t, err, "failed to clean up /tmp/ssh-agent after test") } // BER SSH key is not handled by crypto/ssh diff --git a/prov/ansible/execution.go b/prov/ansible/execution.go index cbb6225eb..c688396b4 100644 --- a/prov/ansible/execution.go +++ b/prov/ansible/execution.go @@ -924,7 +924,7 @@ func (e *executionCommon) executeWithCurrentInstance(ctx context.Context, retry } vaultPassScript := fmt.Sprintf(vaultPassScriptFormat, pythonInterpreter) - if err = ioutil.WriteFile(filepath.Join(ansibleRecipePath, ".vault_pass"), []byte(vaultPassScript), 0764); err != nil { + if err = ioutil.WriteFile(filepath.Join(ansibleRecipePath, ".vault_pass"), []byte(vaultPassScript), 0750); err != nil { err = errors.Wrap(err, "Failed to write .vault_pass file") events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) return err @@ -1203,8 +1203,12 @@ func (e *executionCommon) executePlaybook(ctx context.Context, retry bool, env = append(env, "VAULT_PASSWORD="+e.vaultToken) var sshAgentSocket string if !e.cfg.DisableSSHAgent { + socketDir, err := filepath.Abs(filepath.Join(e.cfg.WorkingDirectory, "ssh-agent")) + if err != nil { + return err + } // Check if SSHAgent is needed - sshAgent, err := e.configureSSHAgent(ctx) + sshAgent, err := e.configureSSHAgent(ctx, socketDir) if err != nil { return errors.Wrap(err, "failed to configure SSH agent for ansible-playbook execution") } @@ -1287,7 +1291,7 @@ func (e *executionCommon) executePlaybook(ctx context.Context, retry bool, return nil } -func (e *executionCommon) configureSSHAgent(ctx context.Context) (*sshutil.SSHAgent, error) { +func (e *executionCommon) configureSSHAgent(ctx context.Context, socketDir string) (*sshutil.SSHAgent, error) { var addSSHAgent bool for _, host := range e.hosts { if len(host.privateKeys) > 0 { @@ -1299,7 +1303,7 @@ func (e *executionCommon) configureSSHAgent(ctx context.Context) (*sshutil.SSHAg return nil, nil } - agent, err := sshutil.NewSSHAgent(ctx) + agent, err := sshutil.NewSSHAgent(ctx, socketDir) if err != nil { return nil, err } @@ -1418,7 +1422,7 @@ func (e *executionCommon) generateAnsibleConfigurationFile( ansibleConfig[ansibleConfigDefaultsHeader]["local_tmp"] = path.Join(config.DefaultSandboxWorkDir, "tmp") // Specify where ansible writes SSH control path sockets in the sandbox container for SSH multiplexing. // (By default, ansible writes to ~/.ansible/cp on host machine and may cause permission denied from inside the container) - // Each sandbox contrainer should have its own control path + // Each sandbox container should have its own control path if e.cfg.Ansible.UseOpenSSH { ansibleConfig[ansibleConfigSSHConnection]["control_path_dir"] = path.Join(config.DefaultSandboxWorkDir, "cp") } diff --git a/prov/ansible/sandbox.go b/prov/ansible/sandbox.go index 8b41db8b0..e26c2edad 100644 --- a/prov/ansible/sandbox.go +++ b/prov/ansible/sandbox.go @@ -65,6 +65,9 @@ func createSandbox(ctx context.Context, cli *client.Client, sandboxCfg *config.D return "", errors.Wrapf(err, "Failed to pull docker image %q", sandboxCfg.Image) } } + // set the home environment for the sandbox container so that ansible resolves its default path (~/.ansible) + // to the work dir inside the container + env = append(env, "HOME="+config.DefaultSandboxWorkDir) env = append(env, sandboxCfg.Env...) cc := &container.Config{ Image: sandboxCfg.Image, @@ -93,6 +96,7 @@ func createSandbox(ctx context.Context, cli *client.Client, sandboxCfg *config.D if err != nil { return "", err } + hc := &container.HostConfig{ AutoRemove: true, // Security hardening for the sandbox container @@ -120,15 +124,18 @@ func createSandbox(ctx context.Context, cli *client.Client, sandboxCfg *config.D Target: config.DefaultSandboxOverlayDir, ReadOnly: true, }, - { - Type: "bind", - Source: sshAgentSocket, - Target: config.DefaultSandboxMountAgentSocket, - ReadOnly: true, - }, }, } + if sshAgentSocket != "" { + hc.Mounts = append(hc.Mounts, mount.Mount{ + Type: "bind", + Source: sshAgentSocket, + Target: config.DefaultSandboxMountAgentSocket, + ReadOnly: true, + }) + } + events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelDEBUG, deploymentID).Registerf("Creating docker container from image: %s", sandboxCfg.Image) createResp, err := cli.ContainerCreate(ctx, cc, hc, nil, "") if err != nil { diff --git a/prov/terraform/executor.go b/prov/terraform/executor.go index bb7b7e9b3..5b8b8c6a2 100644 --- a/prov/terraform/executor.go +++ b/prov/terraform/executor.go @@ -88,7 +88,7 @@ func (e *defaultExecutor) installNode(ctx context.Context, cfg config.Configurat } if !cfg.DisableSSHAgent { - sshAgent, err := sshutil.NewSSHAgent(ctx) + sshAgent, err := sshutil.NewSSHAgent(ctx, "") if err != nil { return err } From 73d0d2f4b4dcacc4696c8d72036315f88002dd8e Mon Sep 17 00:00:00 2001 From: trihoangvo Date: Tue, 9 Jun 2020 11:18:36 +0200 Subject: [PATCH 3/7] Update pull request GH-656 * add new method signature NewSSHAgentWithSocket() * change ansible version in Dockerfile from 2.9.9 to 2.7.9 * remove package scoped variable in getNanoCPUs(), getMemoryInBytes() --- helper/sshutil/sshutil.go | 9 +++++++-- helper/sshutil/sshutil_test.go | 6 +++--- pkg/ansible/Dockerfile | 2 +- prov/ansible/execution.go | 2 +- prov/ansible/sandbox.go | 11 ++--------- prov/terraform/executor.go | 2 +- 6 files changed, 15 insertions(+), 17 deletions(-) diff --git a/helper/sshutil/sshutil.go b/helper/sshutil/sshutil.go index e1a8774c3..ac6766f95 100644 --- a/helper/sshutil/sshutil.go +++ b/helper/sshutil/sshutil.go @@ -17,7 +17,6 @@ package sshutil import ( "bytes" "fmt" - uuid "github.com/satori/go.uuid" "io" "io/ioutil" "net" @@ -31,6 +30,7 @@ import ( "time" "github.com/pkg/errors" + "github.com/satori/go.uuid" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" "golang.org/x/net/context" @@ -269,8 +269,13 @@ func (client *SSHClient) CopyFile(source io.Reader, remotePath string, permissio return nil } +// NewSSHAgent allows to return a new SSH Agent. The agent socket is created at /tmp/ssh-XXXXXXXXXX/agent. by default +func NewSSHAgent(ctx context.Context) (*SSHAgent, error) { + return NewSSHAgentWithSocket(ctx, "") +} + // NewSSHAgent allows to return a new SSH Agent. If socketDir is specified, create the agent socket at /socketDir//agent -func NewSSHAgent(ctx context.Context, socketDir string) (*SSHAgent, error) { +func NewSSHAgentWithSocket(ctx context.Context, socketDir string) (*SSHAgent, error) { bin, err := exec.LookPath("ssh-agent") if err != nil { return nil, errors.Wrap(err, "could not find ssh-agent") diff --git a/helper/sshutil/sshutil_test.go b/helper/sshutil/sshutil_test.go index b0a64f8cd..ed4bf0204 100644 --- a/helper/sshutil/sshutil_test.go +++ b/helper/sshutil/sshutil_test.go @@ -41,7 +41,7 @@ func TestSSHAgent(t *testing.T) { privateKeyContent := string(bArray) // Create new ssh-agent - sshAg, err := NewSSHAgent(context.Background(), "") + sshAg, err := NewSSHAgent(context.Background()) require.Nil(t, err, "unexpected error while creating SSH-agent") defer func() { err = sshAg.Stop() @@ -71,9 +71,9 @@ func TestSSHAgent(t *testing.T) { require.Len(t, keys, 0, "no key expected") // test create more than one agents in a specific socket directory - sshAg1, err := NewSSHAgent(context.Background(), "/tmp/ssh-agent") + sshAg1, err := NewSSHAgentWithSocket(context.Background(), "/tmp/ssh-agent") require.Nil(t, err) - sshAg2, err := NewSSHAgent(context.Background(), "/tmp/ssh-agent") + sshAg2, err := NewSSHAgentWithSocket(context.Background(), "/tmp/ssh-agent") require.Nil(t, err) require.DirExists(t, "/tmp/ssh-agent") assert.NotEqual(t, sshAg1.Socket, sshAg2.Socket) diff --git a/pkg/ansible/Dockerfile b/pkg/ansible/Dockerfile index ebba9b564..54b664125 100644 --- a/pkg/ansible/Dockerfile +++ b/pkg/ansible/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.10 -ENV ANSIBLE_VERSION=2.9.9 +ENV ANSIBLE_VERSION=2.7.9 RUN apk add --no-cache --progress bash python3 openssl ca-certificates openssh sshpass && \ addgroup -g 1000 ansible && \ diff --git a/prov/ansible/execution.go b/prov/ansible/execution.go index c688396b4..90d062a6d 100644 --- a/prov/ansible/execution.go +++ b/prov/ansible/execution.go @@ -1303,7 +1303,7 @@ func (e *executionCommon) configureSSHAgent(ctx context.Context, socketDir strin return nil, nil } - agent, err := sshutil.NewSSHAgent(ctx, socketDir) + agent, err := sshutil.NewSSHAgentWithSocket(ctx, socketDir) if err != nil { return nil, err } diff --git a/prov/ansible/sandbox.go b/prov/ansible/sandbox.go index e26c2edad..0e45e54b5 100644 --- a/prov/ansible/sandbox.go +++ b/prov/ansible/sandbox.go @@ -33,9 +33,6 @@ import ( "github.com/ystia/yorc/v4/log" ) -var nanoCPUs opts.NanoCPUs -var memoryInBytes opts.MemBytes - func createSandbox(ctx context.Context, cli *client.Client, sandboxCfg *config.DockerSandbox, deploymentID, ansibleRecipePath, overlayPath, sshAgentSocket string, env []string) (string, error) { @@ -172,9 +169,7 @@ func stopSandboxOnContextCancellation(ctx context.Context, cli *client.Client, d // getNanoCPUs converts user defined cpus string to nano cpus presentation // defaults cpu to 0.5 cpu if not set func getNanoCPUs(c string) (int64, error) { - if nanoCPUs != 0 { - return nanoCPUs.Value(), nil - } + var nanoCPUs opts.NanoCPUs if c != "" { if err := nanoCPUs.Set(c); err != nil { return 0, err @@ -188,9 +183,7 @@ func getNanoCPUs(c string) (int64, error) { // getMemoryInBytes converts user defined memory string to memory in bytes presentation // defaults memory to 256m if not set func getMemoryInBytes(m string) (int64, error) { - if memoryInBytes != 0 { - return memoryInBytes.Value(), nil - } + var memoryInBytes opts.MemBytes if m != "" { if err := memoryInBytes.Set(m); err != nil { return 0, err diff --git a/prov/terraform/executor.go b/prov/terraform/executor.go index 5b8b8c6a2..bb7b7e9b3 100644 --- a/prov/terraform/executor.go +++ b/prov/terraform/executor.go @@ -88,7 +88,7 @@ func (e *defaultExecutor) installNode(ctx context.Context, cfg config.Configurat } if !cfg.DisableSSHAgent { - sshAgent, err := sshutil.NewSSHAgent(ctx, "") + sshAgent, err := sshutil.NewSSHAgent(ctx) if err != nil { return err } From c69167ccc0fdbeb5a45cc0f7ddbdcc7e325c8637 Mon Sep 17 00:00:00 2001 From: trihoangvo Date: Tue, 9 Jun 2020 11:30:39 +0200 Subject: [PATCH 4/7] Update default sandbox dir --- config/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/config.go b/config/config.go index dccdb049b..d6a869f21 100644 --- a/config/config.go +++ b/config/config.go @@ -85,13 +85,13 @@ const DefaultUpgradesConcurrencyLimit = 1000 const DefaultSSHConnectionTimeout = 10 * time.Second // DefaultSandboxWorkDir is the default workdir in the sandbox container where we mount the ansible recipes -const DefaultSandboxWorkDir = "/home/ansible" +const DefaultSandboxWorkDir = "/work/ansible" // DefaultSandboxOverlayDir is the default directory in the sandbox container where we mount the overlay -const DefaultSandboxOverlayDir = "/home/overlay" +const DefaultSandboxOverlayDir = "/work/overlay" // DefaultSandboxMountAgentSocket is the default location in the sandbox container where we mount the ssh agent socket -const DefaultSandboxMountAgentSocket = "/ssh-agent" +const DefaultSandboxMountAgentSocket = "/work/ssh-agent" // Configuration holds config information filled by Cobra and Viper (see commands package for more information) type Configuration struct { From 972bd3478c4326c507623667e350f4f706c76c08 Mon Sep 17 00:00:00 2001 From: trihoangvo Date: Tue, 9 Jun 2020 11:32:32 +0200 Subject: [PATCH 5/7] Update wrong comment at func NewSSHAgentWithSocket() --- helper/sshutil/sshutil.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper/sshutil/sshutil.go b/helper/sshutil/sshutil.go index ac6766f95..59ff159a0 100644 --- a/helper/sshutil/sshutil.go +++ b/helper/sshutil/sshutil.go @@ -274,7 +274,7 @@ func NewSSHAgent(ctx context.Context) (*SSHAgent, error) { return NewSSHAgentWithSocket(ctx, "") } -// NewSSHAgent allows to return a new SSH Agent. If socketDir is specified, create the agent socket at /socketDir//agent +// NewSSHAgentWithSocket allows to return a new SSH Agent. If socketDir is specified, create the agent socket at /socketDir//agent func NewSSHAgentWithSocket(ctx context.Context, socketDir string) (*SSHAgent, error) { bin, err := exec.LookPath("ssh-agent") if err != nil { From 5c940fcd6c7da317c2aebf68ce118b856b4698a2 Mon Sep 17 00:00:00 2001 From: trihoangvo Date: Thu, 11 Jun 2020 12:26:28 +0200 Subject: [PATCH 6/7] Add netaddr, jmespath in Dockerfile ansible --- pkg/ansible/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ansible/Dockerfile b/pkg/ansible/Dockerfile index 54b664125..32f8d40bc 100644 --- a/pkg/ansible/Dockerfile +++ b/pkg/ansible/Dockerfile @@ -10,6 +10,6 @@ RUN apk add --no-cache --progress bash python3 openssl ca-certificates openssh s if [ ! -e /usr/bin/python ]; then ln -sf /usr/bin/python3 /usr/bin/python; fi && \ apk --update add --virtual build-dependencies python3-dev libffi-dev openssl-dev build-base && \ pip3 install --upgrade pip && \ - pip3 install ansible==${ANSIBLE_VERSION} && \ + pip3 install ansible==${ANSIBLE_VERSION} netaddr jmespath && \ apk del build-dependencies && \ rm -rf /var/cache/apk/* \ No newline at end of file From f0aa1f8366988349f6648a83796e69d8ee9e7f45 Mon Sep 17 00:00:00 2001 From: trihoangvo Date: Sun, 14 Jun 2020 13:08:36 +0200 Subject: [PATCH 7/7] Execution script does not require sandboxing if not hosted op * If an execution script is not a hosted operation, we can execute it directly on the host machine, where yorc is running. * add checkSandboxExecution() to check if the given execution requires sandboxing. Pull request GH-656 --- prov/ansible/execution.go | 28 ++++++++++++++++++++++++---- prov/ansible/execution_ansible.go | 9 +++------ prov/ansible/execution_scripts.go | 4 ++-- prov/ansible/execution_test.go | 5 ++++- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/prov/ansible/execution.go b/prov/ansible/execution.go index 90d062a6d..879908df8 100644 --- a/prov/ansible/execution.go +++ b/prov/ansible/execution.go @@ -174,6 +174,7 @@ type executionCommon struct { isRelationshipTargetNode bool isPerInstanceOperation bool isOrchestratorOperation bool + isSandbox bool IsCustomCommand bool relationshipType string ansibleRunner ansibleRunner @@ -1057,6 +1058,8 @@ func (e *executionCommon) executeWithCurrentInstance(ctx context.Context, retry return err } + checkSandboxExecution(e) + // Generating Ansible config if err = e.generateAnsibleConfigurationFile(ansiblePath, ansibleRecipePath); err != nil { events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) @@ -1215,7 +1218,7 @@ func (e *executionCommon) executePlaybook(ctx context.Context, retry bool, if sshAgent != nil { sshAgentSocket = sshAgent.Socket log.Debugf("Add SSH_AUTH_SOCK env var for ssh-agent") - if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { + if e.isSandbox { env = append(env, "SSH_AUTH_SOCK="+config.DefaultSandboxMountAgentSocket) } else { env = append(env, "SSH_AUTH_SOCK="+sshAgentSocket) @@ -1233,7 +1236,7 @@ func (e *executionCommon) executePlaybook(ctx context.Context, retry bool, } } var cmd *executil.Cmd - if e.cli != nil && e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { + if e.isSandbox { log.Debugf("Start sandbox container with mount directories ansibleRecipePath: %s, overlayPath: %s", ansibleRecipePath, e.OverlayPath) var err error if e.containerID, err = createSandbox(ctx, e.cli, e.cfg.Ansible.HostedOperations.DefaultSandbox, e.deploymentID, @@ -1415,7 +1418,7 @@ func (e *executionCommon) generateAnsibleConfigurationFile( ansibleConfig := getAnsibleConfigFromDefault() // Adding settings whose values are known at runtime, related to the deployment directory path - if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { + if e.isSandbox { ansibleConfig[ansibleConfigDefaultsHeader]["retry_files_save_path"] = config.DefaultSandboxWorkDir // Specify where ansible can write its temp files in the sandbox container. // (By default, ansible writes to ~/.ansible/tmp on host machine and may cause permission denied from inside the container) @@ -1431,7 +1434,7 @@ func (e *executionCommon) generateAnsibleConfigurationFile( } if e.CacheFacts { - if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { + if e.isSandbox { // location in sandbox to save ansible gathering facts // each yorc task should have its own facts (i.e., do not share the gathering facts from different tasks // even in the same deploymentID) so that one task do not overwrite the facts of the other ones unexpectedly @@ -1486,3 +1489,20 @@ func getAnsibleConfigFromDefault() map[string]map[string]string { } return ansibleConfig } + +// checkSandboxExecution check if the given execution requires sandboxing or not. +// A whitelist of unsandbox scenarios are: when unsandbox hosted operation is allowed, when the execution script is not +// a hosted operation, when it is an ansible execution but the sandbox configuration is not provided (for backward +// compatibility) +func checkSandboxExecution(e *executionCommon) { + if e.isOrchestratorOperation && e.cfg.Ansible.HostedOperations.UnsandboxedOperationsAllowed { + return + } + if _, isScript := e.ansibleRunner.(*executionScript); isScript && !e.isOrchestratorOperation { + return + } + if _, isAnsible := e.ansibleRunner.(*executionAnsible); isAnsible && e.cfg.Ansible.HostedOperations.DefaultSandbox == nil { + return + } + e.isSandbox = true +} diff --git a/prov/ansible/execution_ansible.go b/prov/ansible/execution_ansible.go index ef662f305..60e8a1754 100644 --- a/prov/ansible/execution_ansible.go +++ b/prov/ansible/execution_ansible.go @@ -71,9 +71,9 @@ func (e *executionAnsible) generateRunAnsible(ctx context.Context, currentInstan outputHandler := &playbookOutputHandler{} // for operation on host machine, set the ansible destination folder to the ansible recipe path on host machine // for operation on sandbox, set the ansible destination folder and overlay to the default mount path inside the container - var overlayPathOnHost, destFolder string - if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { - overlayPathOnHost = e.OverlayPath + var destFolder string + if e.isSandbox { + overlayPathOnHost := e.OverlayPath e.OverlayPath = config.DefaultSandboxOverlayDir destFolder = config.DefaultSandboxWorkDir defer func() { @@ -211,9 +211,6 @@ func (e *executionAnsible) generateRunAnsible(ctx context.Context, currentInstan events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelERROR, e.deploymentID).RegisterAsString(err.Error()) return outputHandler, err } - if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { - e.OverlayPath = overlayPathOnHost - } events.WithContextOptionalFields(ctx).NewLogEntry(events.LogLevelDEBUG, e.deploymentID).RegisterAsString(fmt.Sprintf("Ansible recipe for node %q: executing %q on remote host(s)", e.NodeName, filepath.Base(e.PlaybookPath))) return &playbookOutputHandler{execution: e, context: ctx}, err diff --git a/prov/ansible/execution_scripts.go b/prov/ansible/execution_scripts.go index 5ec30b4fb..55e4f34da 100644 --- a/prov/ansible/execution_scripts.go +++ b/prov/ansible/execution_scripts.go @@ -213,9 +213,9 @@ func (e *executionScript) generateRunAnsible(ctx context.Context, currentInstanc outputHandler := &scriptOutputHandler{} // for operation on host machine, set the ansible destination folder to the ansible recipe path on host machine // for operation on sandbox, set the ansible destination folder and overlay to the default mount path inside the container - overlayPathOnHost := e.OverlayPath wrapperLocationOnHost := filepath.Join(ansibleRecipePath, "wrapper") - if e.cfg.Ansible.HostedOperations.DefaultSandbox != nil { + if e.isSandbox { + overlayPathOnHost := e.OverlayPath e.OverlayPath = config.DefaultSandboxOverlayDir e.DestFolder = config.DefaultSandboxWorkDir defer func() { diff --git a/prov/ansible/execution_test.go b/prov/ansible/execution_test.go index 57f6e9719..3d42dbb76 100644 --- a/prov/ansible/execution_test.go +++ b/prov/ansible/execution_test.go @@ -744,9 +744,12 @@ func testExecutionGenerateAnsibleConfig(t *testing.T) { len(resultMap[ansibleConfigDefaultsHeader]), "Missing entries in ansible config file with fact caching, content: %q", content) - // Test enabling ansible sandbox, OpenSSH also adds ssh_connection control path + // Test enabling ansible sandbox also add local_tmp in the default section of the ansible config + // OpenSSH also adds control path in the ssh_connection section of the ansible config execution.cfg.Ansible.UseOpenSSH = true execution.cfg.Ansible.HostedOperations.DefaultSandbox = &config.DockerSandbox{} + execution.ansibleRunner = &executionAnsible{} + checkSandboxExecution(execution) err = execution.generateAnsibleConfigurationFile("ansiblePath", yorcConfig.WorkingDirectory) require.NoError(t, err, "Error generating ansible config file") resultMap, content = readAnsibleConfigSettings(t, cfgPath)