From 060585434abb5456935b7378211813b2ceaacaaa Mon Sep 17 00:00:00 2001 From: chris meyers Date: Tue, 19 Mar 2019 18:07:38 -0400 Subject: [PATCH] update tests --- awx/main/tasks.py | 26 +- awx/main/tests/unit/test_tasks.py | 1317 ++++++++++++++-------------- docs/ansible_runner_integration.md | 19 + docs/process_isolation.md | 2 + 4 files changed, 690 insertions(+), 674 deletions(-) create mode 100644 docs/ansible_runner_integration.md diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 9c9665fb65f8..76e1d1ba524c 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -54,7 +54,6 @@ UnifiedJob, Notification, Inventory, SmartInventoryMembership, Job, AdHocCommand, ProjectUpdate, InventoryUpdate, SystemJob, - Project, JobEvent, ProjectUpdateEvent, InventoryUpdateEvent, AdHocCommandEvent, SystemJobEvent, build_safe_env ) @@ -65,8 +64,8 @@ from awx.main.dispatch.publish import task from awx.main.dispatch import get_local_queuename, reaper from awx.main.utils import (get_ssh_version, update_scm_url, - build_proot_temp_dir, get_licenser, - OutputEventFilter, OutputVerboseFilter, ignore_inventory_computed_fields, + get_licenser, + ignore_inventory_computed_fields, ignore_inventory_group_removal, extract_ansible_vars, schedule_task_manager) from awx.main.utils.safe_yaml import safe_dump, sanitize_jinja from awx.main.utils.reload import stop_local_services @@ -961,6 +960,12 @@ def get_password_prompts(self, passwords={}): ''' return OrderedDict() + def create_expect_passwords_data_struct(self, password_prompts, passwords): + expect_passwords = {} + for k, v in password_prompts.items(): + expect_passwords[k] = passwords.get(v, '') or '' + return expect_passwords + def pre_run_hook(self, instance): ''' Hook for any steps to run before the job/task starts @@ -1041,7 +1046,6 @@ def run(self, pk, **kwargs): status, rc, tb = 'error', None, '' output_replacements = [] extra_update_fields = {} - stdout_handle = None fact_modification_times = {} self.event_ct = 0 private_data_dir = None @@ -1099,10 +1103,10 @@ def run(self, pk, **kwargs): ) self.write_args_file(private_data_dir, args) - expect_passwords = {} password_prompts = self.get_password_prompts(passwords) - for k, v in password_prompts.items(): - expect_passwords[k] = passwords.get(v, '') or '' + expect_passwords = self.create_expect_passwords_data_struct(password_prompts, passwords) + + # TODO: Probably remove this when cleaning up isolated path _kw = dict( extra_update_fields=extra_update_fields, proot_cmd=getattr(settings, 'AWX_PROOT_CMD', 'bwrap'), @@ -1369,9 +1373,9 @@ def build_args(self, job, private_data_dir, passwords): ssh_username, become_username, become_method = '', '', '' if creds: - ssh_username = creds.username - become_method = creds.become_method - become_username = creds.become_username + ssh_username = creds.get_input('username', default='') + become_method = creds.get_input('become_method', default='') + become_username = creds.get_input('become_username', default='') else: become_method = None become_username = "" @@ -2140,7 +2144,7 @@ def build_env(self, inventory_update, private_data_dir, isolated, private_data_f 'cloudforms': 'CLOUDFORMS_INI_PATH' } if inventory_update.source in ini_mapping: - cred_data = private_data_files.get('credentials', '') + cred_data = private_data_files.get('credentials', {}) env[ini_mapping[inventory_update.source]] = cred_data.get( inventory_update.get_cloud_credential(), '' ) diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index d3db1667874b..40d0a9a3ab84 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -1,12 +1,8 @@ # -*- coding: utf-8 -*- -from contextlib import contextmanager -from datetime import datetime -from functools import partial import configparser import json import os -import re import shutil import tempfile @@ -15,6 +11,7 @@ from unittest import mock import pytest import yaml +import jinja2 from django.conf import settings @@ -32,19 +29,17 @@ ProjectUpdate, UnifiedJob, User, - Organization, + CustomInventoryScript, build_safe_env ) from awx.main import tasks -from awx.main.queue import CallbackQueueDispatcher -from awx.main.utils import encrypt_field, encrypt_value, OutputEventFilter +from awx.main.utils import encrypt_field, encrypt_value from awx.main.utils.safe_yaml import SafeLoader -from awx.main.exceptions import AwxTaskError class TestJobExecution(object): - pass + EXAMPLE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nxyz==\n-----END PRIVATE KEY-----' @pytest.fixture @@ -426,26 +421,13 @@ class MockMe(): runner_config = MockMe() runner_config.command = {'foo': 'bar'} runner_config.cwd = '/foobar' - runner_config.env = { 'switch': 'blade', 'foot': 'ball' } + runner_config.env = {'switch': 'blade', 'foot': 'ball'} task.status_handler({'status': 'starting'}, runner_config) task.update_model.assert_called_with(1, job_args=json.dumps({'foo': 'bar'}), - job_cwd='/foobar', job_env={'switch': 'blade', 'foot': 'ball'}) + job_cwd='/foobar', job_env={'switch': 'blade', 'foot': 'ball'}) - ''' - def test_artifact_cleanup(self): - path = tempfile.NamedTemporaryFile(delete=False).name - try: - self.task.cleanup_paths.append(path) - assert os.path.exists(path) - self.task.run(self.pk) - assert not os.path.exists(path) - finally: - if os.path.exists(path): - os.remove(path) - ''' - def test_uses_process_isolation(self, settings): job = Job(project=Project(), inventory=Inventory()) task = tasks.RunJob() @@ -544,7 +526,7 @@ def test_invalid_custom_virtualenv(self, patch_Job, private_data_dir): task = tasks.RunJob() with pytest.raises(RuntimeError) as e: - env = task.build_env(job, private_data_dir) + task.build_env(job, private_data_dir) assert 'a valid Python virtualenv does not exist at /venv/missing' == str(e.value) @@ -606,18 +588,20 @@ def test_created_by_extra_vars(self): assert extra_vars['awx_user_name'] == "angry-spud" +@pytest.mark.skip(reason="Isolated code path needs updating after runner integration") class TestIsolatedExecution(TestJobExecution): ISOLATED_HOST = 'some-isolated-host' ISOLATED_CONTROLLER_HOST = 'some-isolated-controller-host' - def get_instance(self): - instance = super(TestIsolatedExecution, self).get_instance() - instance.controller_node = self.ISOLATED_CONTROLLER_HOST - instance.execution_node = self.ISOLATED_HOST - return instance + @pytest.fixture + def job(self): + job = Job(pk=1, id=1, project=Project(), inventory=Inventory(), job_template=JobTemplate(id=1, name='foo')) + job.controller_node = self.ISOLATED_CONTROLLER_HOST + job.execution_node = self.ISOLATED_HOST + return job - def test_with_ssh_credentials(self): + def test_with_ssh_credentials(self, job): ssh = CredentialType.defaults['ssh']() credential = Credential( pk=1, @@ -629,7 +613,7 @@ def test_with_ssh_credentials(self): } ) credential.inputs['password'] = encrypt_field(credential, 'password') - self.instance.credentials.add(credential) + job.credentials.add(credential) private_data = tempfile.mkdtemp(prefix='awx_') self.task.build_private_data_dir = mock.Mock(return_value=private_data) @@ -698,6 +682,32 @@ def _mock_job_artifacts(*args, **kw): class TestJobCredentials(TestJobExecution): + @pytest.fixture + def job(self): + job = Job(pk=1, inventory=Inventory(pk=1), project=Project(pk=1)) + job.websocket_emit_status = mock.Mock() + job._credentials = [] + + credentials_mock = mock.Mock(**{ + 'all': lambda: job._credentials, + 'add': job._credentials.append, + 'filter.return_value': mock.Mock( + __iter__ = lambda *args: iter(job._credentials), + first = lambda: job._credentials[0] + ), + 'spec_set': ['all', 'add', 'filter'] + }) + + with mock.patch.object(UnifiedJob, 'credentials', credentials_mock): + yield job + + @pytest.fixture + def update_model_wrapper(self, job): + def fn(pk, **kwargs): + for k, v in kwargs.items(): + setattr(job, k, v) + return job + return fn parametrize = { 'test_ssh_passwords': [ @@ -707,34 +717,38 @@ class TestJobCredentials(TestJobExecution): ] } - def test_username_jinja_usage(self): + def test_username_jinja_usage(self, job, private_data_dir): + task = tasks.RunJob() ssh = CredentialType.defaults['ssh']() credential = Credential( pk=1, credential_type=ssh, inputs = {'username': '{{ ansible_ssh_pass }}'} ) - self.instance.credentials.add(credential) - with pytest.raises(Exception): - self.task.run(self.pk) - update_model_call = self.task.update_model.call_args[1] - assert 'Jinja variables are not allowed' in update_model_call['result_traceback'] + job.credentials.add(credential) + with pytest.raises(ValueError) as e: + task.build_args(job, private_data_dir, {}) + + assert 'Jinja variables are not allowed' in str(e.value) @pytest.mark.parametrize("flag", ['become_username', 'become_method']) - def test_become_jinja_usage(self, flag): + def test_become_jinja_usage(self, job, private_data_dir, flag): + task = tasks.RunJob() ssh = CredentialType.defaults['ssh']() credential = Credential( pk=1, credential_type=ssh, inputs = {'username': 'joe', flag: '{{ ansible_ssh_pass }}'} ) - self.instance.credentials.add(credential) - with pytest.raises(Exception): - self.task.run(self.pk) - update_model_call = self.task.update_model.call_args[1] - assert 'Jinja variables are not allowed' in update_model_call['result_traceback'] + job.credentials.add(credential) + + with pytest.raises(ValueError) as e: + task.build_args(job, private_data_dir, {}) - def test_ssh_passwords(self, field, password_name, expected_flag): + assert 'Jinja variables are not allowed' in str(e.value) + + def test_ssh_passwords(self, job, private_data_dir, field, password_name, expected_flag): + task = tasks.RunJob() ssh = CredentialType.defaults['ssh']() credential = Credential( pk=1, @@ -742,19 +756,20 @@ def test_ssh_passwords(self, field, password_name, expected_flag): inputs = {'username': 'bob', field: 'secret'} ) credential.inputs[field] = encrypt_field(credential, field) - self.instance.credentials.add(credential) - self.task.run(self.pk) + job.credentials.add(credential) - assert self.run_pexpect.call_count == 1 - call_args, call_kwargs = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + passwords = task.build_passwords(job, {}) + password_prompts = task.get_password_prompts(passwords) + expect_passwords = task.create_expect_passwords_data_struct(password_prompts, passwords) + args = task.build_args(job, private_data_dir, passwords) - assert 'secret' in call_kwargs.get('expect_passwords').values() + assert 'secret' in expect_passwords.values() assert '-u bob' in ' '.join(args) if expected_flag: assert expected_flag in ' '.join(args) - def test_net_ssh_key_unlock(self): + def test_net_ssh_key_unlock(self, job): + task = tasks.RunJob() net = CredentialType.defaults['net']() credential = Credential( pk=1, @@ -762,15 +777,16 @@ def test_net_ssh_key_unlock(self): inputs = {'ssh_key_unlock': 'secret'} ) credential.inputs['ssh_key_unlock'] = encrypt_field(credential, 'ssh_key_unlock') - self.instance.credentials.add(credential) - self.task.run(self.pk) + job.credentials.add(credential) - assert self.run_pexpect.call_count == 1 - call_args, call_kwargs = self.run_pexpect.call_args_list[0] + passwords = task.build_passwords(job, {}) + password_prompts = task.get_password_prompts(passwords) + expect_passwords = task.create_expect_passwords_data_struct(password_prompts, passwords) - assert 'secret' in call_kwargs.get('expect_passwords').values() + assert 'secret' in expect_passwords.values() - def test_net_first_ssh_key_unlock_wins(self): + def test_net_first_ssh_key_unlock_wins(self, job): + task = tasks.RunJob() for i in range(3): net = CredentialType.defaults['net']() credential = Credential( @@ -779,15 +795,16 @@ def test_net_first_ssh_key_unlock_wins(self): inputs = {'ssh_key_unlock': 'secret{}'.format(i)} ) credential.inputs['ssh_key_unlock'] = encrypt_field(credential, 'ssh_key_unlock') - self.instance.credentials.add(credential) - self.task.run(self.pk) + job.credentials.add(credential) - assert self.run_pexpect.call_count == 1 - call_args, call_kwargs = self.run_pexpect.call_args_list[0] + passwords = task.build_passwords(job, {}) + password_prompts = task.get_password_prompts(passwords) + expect_passwords = task.create_expect_passwords_data_struct(password_prompts, passwords) - assert 'secret0' in call_kwargs.get('expect_passwords').values() + assert 'secret0' in expect_passwords.values() - def test_prefer_ssh_over_net_ssh_key_unlock(self): + def test_prefer_ssh_over_net_ssh_key_unlock(self, job): + task = tasks.RunJob() net = CredentialType.defaults['net']() net_credential = Credential( pk=1, @@ -804,16 +821,17 @@ def test_prefer_ssh_over_net_ssh_key_unlock(self): ) ssh_credential.inputs['ssh_key_unlock'] = encrypt_field(ssh_credential, 'ssh_key_unlock') - self.instance.credentials.add(net_credential) - self.instance.credentials.add(ssh_credential) - self.task.run(self.pk) + job.credentials.add(net_credential) + job.credentials.add(ssh_credential) - assert self.run_pexpect.call_count == 1 - call_args, call_kwargs = self.run_pexpect.call_args_list[0] + passwords = task.build_passwords(job, {}) + password_prompts = task.get_password_prompts(passwords) + expect_passwords = task.create_expect_passwords_data_struct(password_prompts, passwords) - assert 'ssh_secret' in call_kwargs.get('expect_passwords').values() + assert 'ssh_secret' in expect_passwords.values() - def test_vault_password(self): + def test_vault_password(self, private_data_dir, job): + task = tasks.RunJob() vault = CredentialType.defaults['vault']() credential = Credential( pk=1, @@ -821,19 +839,18 @@ def test_vault_password(self): inputs={'vault_password': 'vault-me'} ) credential.inputs['vault_password'] = encrypt_field(credential, 'vault_password') - self.instance.credentials.add(credential) - self.task.run(self.pk) + job.credentials.add(credential) - assert self.run_pexpect.call_count == 1 - call_args, call_kwargs = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + passwords = task.build_passwords(job, {}) + args = task.build_args(job, private_data_dir, passwords) + password_prompts = task.get_password_prompts(passwords) + expect_passwords = task.create_expect_passwords_data_struct(password_prompts, passwords) - assert call_kwargs.get('expect_passwords')[ - re.compile(r'Vault password:\s*?$', re.M) - ] == 'vault-me' + assert expect_passwords['Vault password:\s*?$'] == 'vault-me' # noqa assert '--ask-vault-pass' in ' '.join(args) - def test_vault_password_ask(self): + def test_vault_password_ask(self, private_data_dir, job): + task = tasks.RunJob() vault = CredentialType.defaults['vault']() credential = Credential( pk=1, @@ -841,19 +858,18 @@ def test_vault_password_ask(self): inputs={'vault_password': 'ASK'} ) credential.inputs['vault_password'] = encrypt_field(credential, 'vault_password') - self.instance.credentials.add(credential) - self.task.run(self.pk, vault_password='provided-at-launch') + job.credentials.add(credential) - assert self.run_pexpect.call_count == 1 - call_args, call_kwargs = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + passwords = task.build_passwords(job, {'vault_password': 'provided-at-launch'}) + args = task.build_args(job, private_data_dir, passwords) + password_prompts = task.get_password_prompts(passwords) + expect_passwords = task.create_expect_passwords_data_struct(password_prompts, passwords) - assert call_kwargs.get('expect_passwords')[ - re.compile(r'Vault password:\s*?$', re.M) - ] == 'provided-at-launch' + assert expect_passwords['Vault password:\s*?$'] == 'provided-at-launch' # noqa assert '--ask-vault-pass' in ' '.join(args) - def test_multi_vault_password(self): + def test_multi_vault_password(self, private_data_dir, job): + task = tasks.RunJob() vault = CredentialType.defaults['vault']() for i, label in enumerate(['dev', 'prod']): credential = Credential( @@ -862,16 +878,16 @@ def test_multi_vault_password(self): inputs={'vault_password': 'pass@{}'.format(label), 'vault_id': label} ) credential.inputs['vault_password'] = encrypt_field(credential, 'vault_password') - self.instance.credentials.add(credential) - self.task.run(self.pk) + job.credentials.add(credential) - assert self.run_pexpect.call_count == 1 - call_args, call_kwargs = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + passwords = task.build_passwords(job, {}) + args = task.build_args(job, private_data_dir, passwords) + password_prompts = task.get_password_prompts(passwords) + expect_passwords = task.create_expect_passwords_data_struct(password_prompts, passwords) vault_passwords = dict( - (k.pattern, v) for k, v in call_kwargs['expect_passwords'].items() - if 'Vault' in k.pattern + (k, v) for k, v in expect_passwords.items() + if 'Vault' in k ) assert vault_passwords['Vault password \(prod\):\\s*?$'] == 'pass@prod' # noqa assert vault_passwords['Vault password \(dev\):\\s*?$'] == 'pass@dev' # noqa @@ -880,7 +896,8 @@ def test_multi_vault_password(self): assert '--vault-id dev@prompt' in ' '.join(args) assert '--vault-id prod@prompt' in ' '.join(args) - def test_multi_vault_id_conflict(self): + def test_multi_vault_id_conflict(self, job): + task = tasks.RunJob() vault = CredentialType.defaults['vault']() for i in range(2): credential = Credential( @@ -889,12 +906,15 @@ def test_multi_vault_id_conflict(self): inputs={'vault_password': 'some-pass', 'vault_id': 'conflict'} ) credential.inputs['vault_password'] = encrypt_field(credential, 'vault_password') - self.instance.credentials.add(credential) + job.credentials.add(credential) - with pytest.raises(Exception): - self.task.run(self.pk) + with pytest.raises(RuntimeError) as e: + task.build_passwords(job, {}) + + assert 'multiple vault credentials were specified with --vault-id' in str(e.value) - def test_multi_vault_password_ask(self): + def test_multi_vault_password_ask(self, private_data_dir, job): + task = tasks.RunJob() vault = CredentialType.defaults['vault']() for i, label in enumerate(['dev', 'prod']): credential = Credential( @@ -903,19 +923,18 @@ def test_multi_vault_password_ask(self): inputs={'vault_password': 'ASK', 'vault_id': label} ) credential.inputs['vault_password'] = encrypt_field(credential, 'vault_password') - self.instance.credentials.add(credential) - self.task.run(self.pk, **{ + job.credentials.add(credential) + passwords = task.build_passwords(job, { 'vault_password.dev': 'provided-at-launch@dev', 'vault_password.prod': 'provided-at-launch@prod' }) - - assert self.run_pexpect.call_count == 1 - call_args, call_kwargs = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + args = task.build_args(job, private_data_dir, passwords) + password_prompts = task.get_password_prompts(passwords) + expect_passwords = task.create_expect_passwords_data_struct(password_prompts, passwords) vault_passwords = dict( - (k.pattern, v) for k, v in call_kwargs['expect_passwords'].items() - if 'Vault' in k.pattern + (k, v) for k, v in expect_passwords.items() + if 'Vault' in k ) assert vault_passwords['Vault password \(prod\):\\s*?$'] == 'provided-at-launch@prod' # noqa assert vault_passwords['Vault password \(dev\):\\s*?$'] == 'provided-at-launch@dev' # noqa @@ -924,38 +943,7 @@ def test_multi_vault_password_ask(self): assert '--vault-id dev@prompt' in ' '.join(args) assert '--vault-id prod@prompt' in ' '.join(args) - def test_ssh_key_with_agent(self): - ssh = CredentialType.defaults['ssh']() - credential = Credential( - pk=1, - credential_type=ssh, - inputs = { - 'username': 'bob', - 'ssh_key_data': self.EXAMPLE_PRIVATE_KEY - } - ) - credential.inputs['ssh_key_data'] = encrypt_field(credential, 'ssh_key_data') - self.instance.credentials.add(credential) - - def run_pexpect_side_effect(private_data, *args, **kwargs): - args, cwd, env, stdout = args - ssh_key_data_fifo = '/'.join([private_data, 'credential_1']) - assert open(ssh_key_data_fifo, 'r').read() == self.EXAMPLE_PRIVATE_KEY - assert ' '.join(args).startswith( - 'ssh-agent -a %s sh -c ssh-add %s && rm -f %s' % ( - '/'.join([private_data, 'ssh_auth.sock']), - ssh_key_data_fifo, - ssh_key_data_fifo - ) - ) - return ['successful', 0] - - private_data = tempfile.mkdtemp(prefix='awx_') - self.task.build_private_data_dir = mock.Mock(return_value=private_data) - self.run_pexpect.side_effect = partial(run_pexpect_side_effect, private_data) - self.task.run(self.pk, private_data_dir=private_data) - - def test_aws_cloud_credential(self): + def test_aws_cloud_credential(self, job, private_data_dir): aws = CredentialType.defaults['aws']() credential = Credential( pk=1, @@ -963,19 +951,20 @@ def test_aws_cloud_credential(self): inputs = {'username': 'bob', 'password': 'secret'} ) credential.inputs['password'] = encrypt_field(credential, 'password') - self.instance.credentials.add(credential) - self.task.run(self.pk) + job.credentials.add(credential) - assert self.run_pexpect.call_count == 1 - call_args, _ = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + env = {} + safe_env = {} + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) assert env['AWS_ACCESS_KEY_ID'] == 'bob' assert env['AWS_SECRET_ACCESS_KEY'] == 'secret' assert 'AWS_SECURITY_TOKEN' not in env - assert self.instance.job_env['AWS_SECRET_ACCESS_KEY'] == tasks.HIDDEN_PASSWORD + assert safe_env['AWS_SECRET_ACCESS_KEY'] == tasks.HIDDEN_PASSWORD - def test_aws_cloud_credential_with_sts_token(self): + def test_aws_cloud_credential_with_sts_token(self, private_data_dir, job): aws = CredentialType.defaults['aws']() credential = Credential( pk=1, @@ -984,19 +973,20 @@ def test_aws_cloud_credential_with_sts_token(self): ) for key in ('password', 'security_token'): credential.inputs[key] = encrypt_field(credential, key) - self.instance.credentials.add(credential) - self.task.run(self.pk) + job.credentials.add(credential) - assert self.run_pexpect.call_count == 1 - call_args, _ = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + env = {} + safe_env = {} + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) assert env['AWS_ACCESS_KEY_ID'] == 'bob' assert env['AWS_SECRET_ACCESS_KEY'] == 'secret' assert env['AWS_SECURITY_TOKEN'] == 'token' - assert self.instance.job_env['AWS_SECRET_ACCESS_KEY'] == tasks.HIDDEN_PASSWORD + assert safe_env['AWS_SECRET_ACCESS_KEY'] == tasks.HIDDEN_PASSWORD - def test_gce_credentials(self): + def test_gce_credentials(self, private_data_dir, job): gce = CredentialType.defaults['gce']() credential = Credential( pk=1, @@ -1008,21 +998,20 @@ def test_gce_credentials(self): } ) credential.inputs['ssh_key_data'] = encrypt_field(credential, 'ssh_key_data') - self.instance.credentials.add(credential) - - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - json_data = json.load(open(env['GCE_CREDENTIALS_FILE_PATH'], 'rb')) - assert json_data['type'] == 'service_account' - assert json_data['private_key'] == self.EXAMPLE_PRIVATE_KEY - assert json_data['client_email'] == 'bob' - assert json_data['project_id'] == 'some-project' - return ['successful', 0] + job.credentials.add(credential) - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + env = {} + safe_env = {} + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) + json_data = json.load(open(env['GCE_CREDENTIALS_FILE_PATH'], 'rb')) + assert json_data['type'] == 'service_account' + assert json_data['private_key'] == self.EXAMPLE_PRIVATE_KEY + assert json_data['client_email'] == 'bob' + assert json_data['project_id'] == 'some-project' - def test_azure_rm_with_tenant(self): + def test_azure_rm_with_tenant(self, private_data_dir, job): azure = CredentialType.defaults['azure_rm']() credential = Credential( pk=1, @@ -1035,21 +1024,21 @@ def test_azure_rm_with_tenant(self): } ) credential.inputs['secret'] = encrypt_field(credential, 'secret') - self.instance.credentials.add(credential) - - self.task.run(self.pk) + job.credentials.add(credential) - assert self.run_pexpect.call_count == 1 - call_args, _ = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + env = {} + safe_env = {} + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) assert env['AZURE_CLIENT_ID'] == 'some-client' assert env['AZURE_SECRET'] == 'some-secret' assert env['AZURE_TENANT'] == 'some-tenant' assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription' - assert self.instance.job_env['AZURE_SECRET'] == tasks.HIDDEN_PASSWORD + assert safe_env['AZURE_SECRET'] == tasks.HIDDEN_PASSWORD - def test_azure_rm_with_password(self): + def test_azure_rm_with_password(self, private_data_dir, job): azure = CredentialType.defaults['azure_rm']() credential = Credential( pk=1, @@ -1062,21 +1051,21 @@ def test_azure_rm_with_password(self): } ) credential.inputs['password'] = encrypt_field(credential, 'password') - self.instance.credentials.add(credential) - - self.task.run(self.pk) + job.credentials.add(credential) - assert self.run_pexpect.call_count == 1 - call_args, _ = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + env = {} + safe_env = {} + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription' assert env['AZURE_AD_USER'] == 'bob' assert env['AZURE_PASSWORD'] == 'secret' assert env['AZURE_CLOUD_ENVIRONMENT'] == 'foobar' - assert self.instance.job_env['AZURE_PASSWORD'] == tasks.HIDDEN_PASSWORD + assert safe_env['AZURE_PASSWORD'] == tasks.HIDDEN_PASSWORD - def test_vmware_credentials(self): + def test_vmware_credentials(self, private_data_dir, job): vmware = CredentialType.defaults['vmware']() credential = Credential( pk=1, @@ -1084,19 +1073,21 @@ def test_vmware_credentials(self): inputs = {'username': 'bob', 'password': 'secret', 'host': 'https://example.org'} ) credential.inputs['password'] = encrypt_field(credential, 'password') - self.instance.credentials.add(credential) - self.task.run(self.pk) + job.credentials.add(credential) - assert self.run_pexpect.call_count == 1 - call_args, _ = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + env = {} + safe_env = {} + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) assert env['VMWARE_USER'] == 'bob' assert env['VMWARE_PASSWORD'] == 'secret' assert env['VMWARE_HOST'] == 'https://example.org' - assert self.instance.job_env['VMWARE_PASSWORD'] == tasks.HIDDEN_PASSWORD + assert safe_env['VMWARE_PASSWORD'] == tasks.HIDDEN_PASSWORD - def test_openstack_credentials(self): + def test_openstack_credentials(self, private_data_dir, job): + task = tasks.RunJob() openstack = CredentialType.defaults['openstack']() credential = Credential( pk=1, @@ -1109,29 +1100,29 @@ def test_openstack_credentials(self): } ) credential.inputs['password'] = encrypt_field(credential, 'password') - self.instance.credentials.add(credential) + job.credentials.add(credential) - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - shade_config = open(env['OS_CLIENT_CONFIG_FILE'], 'r').read() - assert shade_config == '\n'.join([ - 'clouds:', - ' devstack:', - ' auth:', - ' auth_url: https://keystone.example.org', - ' password: secret', - ' project_name: tenant-name', - ' username: bob', - ' verify: true', - '' - ]) - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + private_data_files = task.build_private_data_files(job, private_data_dir) + env = task.build_env(job, private_data_dir, private_data_files=private_data_files) + credential.credential_type.inject_credential( + credential, env, {}, [], private_data_dir + ) + + shade_config = open(env['OS_CLIENT_CONFIG_FILE'], 'r').read() + assert shade_config == '\n'.join([ + 'clouds:', + ' devstack:', + ' auth:', + ' auth_url: https://keystone.example.org', + ' password: secret', + ' project_name: tenant-name', + ' username: bob', + ' verify: true', + '' + ]) @pytest.mark.parametrize("ca_file", [None, '/path/to/some/file']) - def test_rhv_credentials(self, ca_file): + def test_rhv_credentials(self, private_data_dir, job, ca_file): rhv = CredentialType.defaults['rhv']() inputs = { 'host': 'some-ovirt-host.example.org', @@ -1146,31 +1137,32 @@ def test_rhv_credentials(self, ca_file): inputs=inputs ) credential.inputs['password'] = encrypt_field(credential, 'password') - self.instance.credentials.add(credential) + job.credentials.add(credential) - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - config = configparser.ConfigParser() - config.read(env['OVIRT_INI_PATH']) - assert config.get('ovirt', 'ovirt_url') == 'some-ovirt-host.example.org' - assert config.get('ovirt', 'ovirt_username') == 'bob' - assert config.get('ovirt', 'ovirt_password') == 'some-pass' - if ca_file: - assert config.get('ovirt', 'ovirt_ca_file') == ca_file - else: - with pytest.raises(configparser.NoOptionError): - config.get('ovirt', 'ovirt_ca_file') - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + env = {} + safe_env = {} + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) + + config = configparser.ConfigParser() + config.read(env['OVIRT_INI_PATH']) + assert config.get('ovirt', 'ovirt_url') == 'some-ovirt-host.example.org' + assert config.get('ovirt', 'ovirt_username') == 'bob' + assert config.get('ovirt', 'ovirt_password') == 'some-pass' + if ca_file: + assert config.get('ovirt', 'ovirt_ca_file') == ca_file + else: + with pytest.raises(configparser.NoOptionError): + config.get('ovirt', 'ovirt_ca_file') @pytest.mark.parametrize('authorize, expected_authorize', [ [True, '1'], [False, '0'], [None, '0'], ]) - def test_net_credentials(self, authorize, expected_authorize): + def test_net_credentials(self, authorize, expected_authorize, job, private_data_dir): + task = tasks.RunJob() net = CredentialType.defaults['net']() inputs = { 'username': 'bob', @@ -1180,26 +1172,27 @@ def test_net_credentials(self, authorize, expected_authorize): } if authorize is not None: inputs['authorize'] = authorize - credential = Credential(pk=1,credential_type=net, inputs = inputs) + credential = Credential(pk=1, credential_type=net, inputs=inputs) for field in ('password', 'ssh_key_data', 'authorize_password'): credential.inputs[field] = encrypt_field(credential, field) - self.instance.credentials.add(credential) + job.credentials.add(credential) - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - assert env['ANSIBLE_NET_USERNAME'] == 'bob' - assert env['ANSIBLE_NET_PASSWORD'] == 'secret' - assert env['ANSIBLE_NET_AUTHORIZE'] == expected_authorize - if authorize: - assert env['ANSIBLE_NET_AUTH_PASS'] == 'authorizeme' - assert open(env['ANSIBLE_NET_SSH_KEYFILE'], 'r').read() == self.EXAMPLE_PRIVATE_KEY - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) - assert self.instance.job_env['ANSIBLE_NET_PASSWORD'] == tasks.HIDDEN_PASSWORD + private_data_files = task.build_private_data_files(job, private_data_dir) + env = task.build_env(job, private_data_dir, private_data_files=private_data_files) + safe_env = build_safe_env(env) + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) + + assert env['ANSIBLE_NET_USERNAME'] == 'bob' + assert env['ANSIBLE_NET_PASSWORD'] == 'secret' + assert env['ANSIBLE_NET_AUTHORIZE'] == expected_authorize + if authorize: + assert env['ANSIBLE_NET_AUTH_PASS'] == 'authorizeme' + assert open(env['ANSIBLE_NET_SSH_KEYFILE'], 'r').read() == self.EXAMPLE_PRIVATE_KEY + assert safe_env['ANSIBLE_NET_PASSWORD'] == tasks.HIDDEN_PASSWORD - def test_custom_environment_injectors_with_jinja_syntax_error(self): + def test_custom_environment_injectors_with_jinja_syntax_error(self, private_data_dir): some_cloud = CredentialType( kind='cloud', name='SomeCloud', @@ -1222,11 +1215,13 @@ def test_custom_environment_injectors_with_jinja_syntax_error(self): credential_type=some_cloud, inputs = {'api_token': 'ABC123'} ) - self.instance.credentials.add(credential) - with pytest.raises(Exception): - self.task.run(self.pk) - def test_custom_environment_injectors(self): + with pytest.raises(jinja2.exceptions.UndefinedError): + credential.credential_type.inject_credential( + credential, {}, {}, [], private_data_dir + ) + + def test_custom_environment_injectors(self, private_data_dir): some_cloud = CredentialType( kind='cloud', name='SomeCloud', @@ -1249,16 +1244,15 @@ def test_custom_environment_injectors(self): credential_type=some_cloud, inputs = {'api_token': 'ABC123'} ) - self.instance.credentials.add(credential) - self.task.run(self.pk) - assert self.run_pexpect.call_count == 1 - call_args, _ = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + env = {} + credential.credential_type.inject_credential( + credential, env, {}, [], private_data_dir + ) assert env['MY_CLOUD_API_TOKEN'] == 'ABC123' - def test_custom_environment_injectors_with_boolean_env_var(self): + def test_custom_environment_injectors_with_boolean_env_var(self, private_data_dir): some_cloud = CredentialType( kind='cloud', name='SomeCloud', @@ -1281,15 +1275,16 @@ def test_custom_environment_injectors_with_boolean_env_var(self): credential_type=some_cloud, inputs={'turbo_button': True} ) - self.instance.credentials.add(credential) - self.task.run(self.pk) - assert self.run_pexpect.call_count == 1 - call_args, _ = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + env = {} + credential.credential_type.inject_credential( + credential, env, {}, [], private_data_dir + ) + assert env['TURBO_BUTTON'] == str(True) - def test_custom_environment_injectors_with_reserved_env_var(self): + def test_custom_environment_injectors_with_reserved_env_var(self, private_data_dir, job): + task = tasks.RunJob() some_cloud = CredentialType( kind='cloud', name='SomeCloud', @@ -1312,16 +1307,13 @@ def test_custom_environment_injectors_with_reserved_env_var(self): credential_type=some_cloud, inputs = {'api_token': 'ABC123'} ) - self.instance.credentials.add(credential) - self.task.run(self.pk) + job.credentials.add(credential) - assert self.run_pexpect.call_count == 1 - call_args, _ = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + env = task.build_env(job, private_data_dir) - assert env['JOB_ID'] == str(self.instance.pk) + assert env['JOB_ID'] == str(job.pk) - def test_custom_environment_injectors_with_secret_field(self): + def test_custom_environment_injectors_with_secret_field(self, private_data_dir): some_cloud = CredentialType( kind='cloud', name='SomeCloud', @@ -1346,18 +1338,19 @@ def test_custom_environment_injectors_with_secret_field(self): inputs = {'password': 'SUPER-SECRET-123'} ) credential.inputs['password'] = encrypt_field(credential, 'password') - self.instance.credentials.add(credential) - self.task.run(self.pk) - assert self.run_pexpect.call_count == 1 - call_args, _ = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + env = {} + safe_env = {} + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) assert env['MY_CLOUD_PRIVATE_VAR'] == 'SUPER-SECRET-123' - assert 'SUPER-SECRET-123' not in json.dumps(self.task.update_model.call_args_list) - assert self.instance.job_env['MY_CLOUD_PRIVATE_VAR'] == tasks.HIDDEN_PASSWORD + assert 'SUPER-SECRET-123' not in safe_env.values() + assert safe_env['MY_CLOUD_PRIVATE_VAR'] == tasks.HIDDEN_PASSWORD - def test_custom_environment_injectors_with_extra_vars(self): + def test_custom_environment_injectors_with_extra_vars(self, private_data_dir, job): + task = tasks.RunJob() some_cloud = CredentialType( kind='cloud', name='SomeCloud', @@ -1380,19 +1373,19 @@ def test_custom_environment_injectors_with_extra_vars(self): credential_type=some_cloud, inputs = {'api_token': 'ABC123'} ) - self.instance.credentials.add(credential) + job.credentials.add(credential) - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - extra_vars = parse_extra_vars(args) - assert extra_vars["api_token"] == "ABC123" - assert hasattr(extra_vars["api_token"], '__UNSAFE__') - return ['successful', 0] + args = task.build_args(job, private_data_dir, {}) + credential.credential_type.inject_credential( + credential, {}, {}, args, private_data_dir + ) + extra_vars = parse_extra_vars(args) - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + assert extra_vars["api_token"] == "ABC123" + assert hasattr(extra_vars["api_token"], '__UNSAFE__') - def test_custom_environment_injectors_with_boolean_extra_vars(self): + def test_custom_environment_injectors_with_boolean_extra_vars(self, job, private_data_dir): + task = tasks.RunJob() some_cloud = CredentialType( kind='cloud', name='SomeCloud', @@ -1415,18 +1408,19 @@ def test_custom_environment_injectors_with_boolean_extra_vars(self): credential_type=some_cloud, inputs={'turbo_button': True} ) - self.instance.credentials.add(credential) + job.credentials.add(credential) - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - extra_vars = parse_extra_vars(args) - assert extra_vars["turbo_button"] == "True" - return ['successful', 0] + args = task.build_args(job, private_data_dir, {}) + credential.credential_type.inject_credential( + credential, {}, {}, args, private_data_dir + ) + extra_vars = parse_extra_vars(args) - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + assert extra_vars["turbo_button"] == "True" + return ['successful', 0] - def test_custom_environment_injectors_with_complicated_boolean_template(self): + def test_custom_environment_injectors_with_complicated_boolean_template(self, job, private_data_dir): + task = tasks.RunJob() some_cloud = CredentialType( kind='cloud', name='SomeCloud', @@ -1449,21 +1443,21 @@ def test_custom_environment_injectors_with_complicated_boolean_template(self): credential_type=some_cloud, inputs={'turbo_button': True} ) - self.instance.credentials.add(credential) + job.credentials.add(credential) - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - extra_vars = parse_extra_vars(args) - assert extra_vars["turbo_button"] == "FAST!" - return ['successful', 0] + args = task.build_args(job, private_data_dir, {}) + credential.credential_type.inject_credential( + credential, {}, {}, args, private_data_dir + ) + extra_vars = parse_extra_vars(args) - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + assert extra_vars["turbo_button"] == "FAST!" - def test_custom_environment_injectors_with_secret_extra_vars(self): + def test_custom_environment_injectors_with_secret_extra_vars(self, job, private_data_dir): """ extra_vars that contain secret field values should be censored in the DB """ + task = tasks.RunJob() some_cloud = CredentialType( kind='cloud', name='SomeCloud', @@ -1488,20 +1482,17 @@ def test_custom_environment_injectors_with_secret_extra_vars(self): inputs = {'password': 'SUPER-SECRET-123'} ) credential.inputs['password'] = encrypt_field(credential, 'password') - self.instance.credentials.add(credential) + job.credentials.add(credential) - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - extra_vars = parse_extra_vars(args) - assert extra_vars["password"] == "SUPER-SECRET-123" - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + args = task.build_args(job, private_data_dir, {}) + credential.credential_type.inject_credential( + credential, {}, {}, args, private_data_dir + ) - assert 'SUPER-SECRET-123' not in json.dumps(self.task.update_model.call_args_list) + extra_vars = parse_extra_vars(args) + assert extra_vars["password"] == "SUPER-SECRET-123" - def test_custom_environment_injectors_with_file(self): + def test_custom_environment_injectors_with_file(self, private_data_dir): some_cloud = CredentialType( kind='cloud', name='SomeCloud', @@ -1527,18 +1518,15 @@ def test_custom_environment_injectors_with_file(self): credential_type=some_cloud, inputs = {'api_token': 'ABC123'} ) - self.instance.credentials.add(credential) - self.task.run(self.pk) - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - assert open(env['MY_CLOUD_INI_FILE'], 'r').read() == '[mycloud]\nABC123' - return ['successful', 0] + env = {} + credential.credential_type.inject_credential( + credential, env, {}, [], private_data_dir + ) - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + assert open(env['MY_CLOUD_INI_FILE'], 'r').read() == '[mycloud]\nABC123' - def test_custom_environment_injectors_with_unicode_content(self): + def test_custom_environment_injectors_with_unicode_content(self, private_data_dir): value = 'Iñtërnâtiônàlizætiøn' some_cloud = CredentialType( kind='cloud', @@ -1554,18 +1542,15 @@ def test_custom_environment_injectors_with_unicode_content(self): pk=1, credential_type=some_cloud, ) - self.instance.credentials.add(credential) - self.task.run(self.pk) - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - assert open(env['MY_CLOUD_INI_FILE'], 'r').read() == value - return ['successful', 0] + env = {} + credential.credential_type.inject_credential( + credential, env, {}, [], private_data_dir + ) - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + assert open(env['MY_CLOUD_INI_FILE'], 'r').read() == value - def test_custom_environment_injectors_with_files(self): + def test_custom_environment_injectors_with_files(self, private_data_dir): some_cloud = CredentialType( kind='cloud', name='SomeCloud', @@ -1597,19 +1582,16 @@ def test_custom_environment_injectors_with_files(self): credential_type=some_cloud, inputs = {'cert': 'CERT123', 'key': 'KEY123'} ) - self.instance.credentials.add(credential) - self.task.run(self.pk) - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - assert open(env['MY_CERT_INI_FILE'], 'r').read() == '[mycert]\nCERT123' - assert open(env['MY_KEY_INI_FILE'], 'r').read() == '[mykey]\nKEY123' - return ['successful', 0] + env = {} + credential.credential_type.inject_credential( + credential, env, {}, [], private_data_dir + ) - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + assert open(env['MY_CERT_INI_FILE'], 'r').read() == '[mycert]\nCERT123' + assert open(env['MY_KEY_INI_FILE'], 'r').read() == '[mykey]\nKEY123' - def test_multi_cloud(self): + def test_multi_cloud(self, private_data_dir): gce = CredentialType.defaults['gce']() gce_credential = Credential( pk=1, @@ -1617,11 +1599,10 @@ def test_multi_cloud(self): inputs = { 'username': 'bob', 'project': 'some-project', - 'ssh_key_data': 'GCE: %s' % self.EXAMPLE_PRIVATE_KEY + 'ssh_key_data': self.EXAMPLE_PRIVATE_KEY } ) gce_credential.inputs['ssh_key_data'] = encrypt_field(gce_credential, 'ssh_key_data') - self.instance.credentials.add(gce_credential) azure_rm = CredentialType.defaults['azure_rm']() azure_rm_credential = Credential( @@ -1635,40 +1616,40 @@ def test_multi_cloud(self): ) azure_rm_credential.inputs['secret'] = '' azure_rm_credential.inputs['secret'] = encrypt_field(azure_rm_credential, 'secret') - self.instance.credentials.add(azure_rm_credential) - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args + env = {} + safe_env = {} + for credential in [gce_credential, azure_rm_credential]: + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) - assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription' - assert env['AZURE_AD_USER'] == 'bob' - assert env['AZURE_PASSWORD'] == 'secret' + assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription' + assert env['AZURE_AD_USER'] == 'bob' + assert env['AZURE_PASSWORD'] == 'secret' - return ['successful', 0] + json_data = json.load(open(env['GCE_CREDENTIALS_FILE_PATH'], 'rb')) + assert json_data['type'] == 'service_account' + assert json_data['private_key'] == self.EXAMPLE_PRIVATE_KEY + assert json_data['client_email'] == 'bob' + assert json_data['project_id'] == 'some-project' - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) - assert self.instance.job_env['AZURE_PASSWORD'] == tasks.HIDDEN_PASSWORD + assert safe_env['AZURE_PASSWORD'] == tasks.HIDDEN_PASSWORD - def test_awx_task_env(self): - with mock.patch('awx.main.tasks.settings.AWX_TASK_ENV', {'FOO': 'BAR'}): - self.task.run(self.pk) + def test_awx_task_env(self, settings, private_data_dir, job): + settings.AWX_TASK_ENV = {'FOO': 'BAR'} + task = tasks.RunJob() + env = task.build_env(job, private_data_dir) - assert self.run_pexpect.call_count == 1 - call_args, _ = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args assert env['FOO'] == 'BAR' class TestProjectUpdateCredentials(TestJobExecution): - - TASK_CLS = tasks.RunProjectUpdate - - def get_instance(self): - return ProjectUpdate( - pk=1, - project=Project() - ) + @pytest.fixture + def project_update(self): + project_update = ProjectUpdate(pk=1, project=Project(pk=1)) + project_update.websocket_emit_status = mock.Mock() + return project_update parametrize = { 'test_username_and_password_auth': [ @@ -1688,53 +1669,53 @@ def get_instance(self): ] } - def test_bwrap_exposes_projects_root(self): + def test_process_isolation_exposes_projects_root(self, private_data_dir, project_update): + task = tasks.RunProjectUpdate() + task.revision_path = 'foobar' ssh = CredentialType.defaults['ssh']() - self.instance.scm_type = 'git' - self.instance.credential = Credential( + project_update.scm_type = 'git' + project_update.credential = Credential( pk=1, credential_type=ssh, ) + process_isolation = task.build_params_process_isolation(job, private_data_dir, 'cwd') - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - extra_vars = parse_extra_vars(args) - assert ' '.join(args).startswith('bwrap') - assert ' '.join([ - '--bind', - os.path.realpath(settings.PROJECTS_ROOT), - os.path.realpath(settings.PROJECTS_ROOT) - ]) in ' '.join(args) - assert extra_vars["scm_revision_output"].startswith(settings.PROJECTS_ROOT) - return ['successful', 0] + assert process_isolation['process_isolation'] is True + assert settings.PROJECTS_ROOT in process_isolation['process_isolation_show_paths'] - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + task._write_extra_vars_file = mock.Mock() + task.build_extra_vars_file(project_update, private_data_dir, {}) - def test_username_and_password_auth(self, scm_type): + call_args, _ = task._write_extra_vars_file.call_args_list[0] + _, extra_vars = call_args + + assert extra_vars["scm_revision_output"] == 'foobar' + + def test_username_and_password_auth(self, project_update, scm_type): + task = tasks.RunProjectUpdate() ssh = CredentialType.defaults['ssh']() - self.instance.scm_type = scm_type - self.instance.credential = Credential( + project_update.scm_type = scm_type + project_update.credential = Credential( pk=1, credential_type=ssh, inputs = {'username': 'bob', 'password': 'secret'} ) - self.instance.credential.inputs['password'] = encrypt_field( - self.instance.credential, 'password' + project_update.credential.inputs['password'] = encrypt_field( + project_update.credential, 'password' ) - self.task.run(self.pk) - assert self.run_pexpect.call_count == 1 - call_args, call_kwargs = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args + passwords = task.build_passwords(project_update, {}) + password_prompts = task.get_password_prompts(passwords) + expect_passwords = task.create_expect_passwords_data_struct(password_prompts, passwords) - assert 'bob' in call_kwargs.get('expect_passwords').values() - assert 'secret' in call_kwargs.get('expect_passwords').values() + assert 'bob' in expect_passwords.values() + assert 'secret' in expect_passwords.values() - def test_ssh_key_auth(self, scm_type): + def test_ssh_key_auth(self, project_update, scm_type): + task = tasks.RunProjectUpdate() ssh = CredentialType.defaults['ssh']() - self.instance.scm_type = scm_type - self.instance.credential = Credential( + project_update.scm_type = scm_type + project_update.credential = Credential( pk=1, credential_type=ssh, inputs = { @@ -1742,45 +1723,28 @@ def test_ssh_key_auth(self, scm_type): 'ssh_key_data': self.EXAMPLE_PRIVATE_KEY } ) - self.instance.credential.inputs['ssh_key_data'] = encrypt_field( - self.instance.credential, 'ssh_key_data' + project_update.credential.inputs['ssh_key_data'] = encrypt_field( + project_update.credential, 'ssh_key_data' ) - def run_pexpect_side_effect(private_data, *args, **kwargs): - args, cwd, env, stdout = args - ssh_key_data_fifo = '/'.join([private_data, 'credential_1']) - assert open(ssh_key_data_fifo, 'r').read() == self.EXAMPLE_PRIVATE_KEY - assert ' '.join(args).startswith( - 'ssh-agent -a %s sh -c ssh-add %s && rm -f %s' % ( - '/'.join([private_data, 'ssh_auth.sock']), - ssh_key_data_fifo, - ssh_key_data_fifo - ) - ) - assert 'bob' in kwargs.get('expect_passwords').values() - return ['successful', 0] + passwords = task.build_passwords(project_update, {}) + password_prompts = task.get_password_prompts(passwords) + expect_passwords = task.create_expect_passwords_data_struct(password_prompts, passwords) + assert 'bob' in expect_passwords.values() - private_data = tempfile.mkdtemp(prefix='awx_') - self.task.build_private_data_dir = mock.Mock(return_value=private_data) - self.run_pexpect.side_effect = partial(run_pexpect_side_effect, private_data) - self.task.run(self.pk) + def test_awx_task_env(self, project_update, settings, private_data_dir, scm_type): + settings.AWX_TASK_ENV = {'FOO': 'BAR'} + task = tasks.RunProjectUpdate() + project_update.scm_type = scm_type - def test_awx_task_env(self, scm_type): - self.instance.scm_type = scm_type - with mock.patch('awx.main.tasks.settings.AWX_TASK_ENV', {'FOO': 'BAR'}): - self.task.run(self.pk) + env = task.build_env(project_update, private_data_dir) - assert self.run_pexpect.call_count == 1 - call_args, _ = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args assert env['FOO'] == 'BAR' class TestInventoryUpdateCredentials(TestJobExecution): - - TASK_CLS = tasks.RunInventoryUpdate - - def get_instance(self): + @pytest.fixture + def inventory_update(self): return InventoryUpdate( pk=1, inventory_source=InventorySource( @@ -1789,34 +1753,28 @@ def get_instance(self): ) ) - def test_source_without_credential(self, mocker): - self.instance.source = 'ec2' - self.instance.get_cloud_credential = mocker.Mock(return_value=None) + def test_source_without_credential(self, mocker, inventory_update, private_data_dir): + task = tasks.RunInventoryUpdate() + inventory_update.source = 'ec2' + inventory_update.get_cloud_credential = mocker.Mock(return_value=None) - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args + private_data_files = task.build_private_data_files(inventory_update, private_data_dir) + env = task.build_env(inventory_update, private_data_dir, False, private_data_files) - assert 'AWS_ACCESS_KEY_ID' not in env - assert 'AWS_SECRET_ACCESS_KEY' not in env - assert 'EC2_INI_PATH' in env + assert 'AWS_ACCESS_KEY_ID' not in env + assert 'AWS_SECRET_ACCESS_KEY' not in env + assert 'EC2_INI_PATH' in env - config = configparser.ConfigParser() - config.read(env['EC2_INI_PATH']) - assert 'ec2' in config.sections() - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + config = configparser.ConfigParser() + config.read(env['EC2_INI_PATH']) + assert 'ec2' in config.sections() @pytest.mark.parametrize('with_credential', [True, False]) - def test_custom_source(self, with_credential, mocker): - self.instance.source = 'custom' - self.instance.source_vars = '{"FOO": "BAR"}' - patch = mock.patch.object(InventoryUpdate, 'source_script', mock.Mock( - script='#!/bin/sh\necho "Hello, World!"') - ) - self.patches.append(patch) - patch.start() + def test_custom_source(self, with_credential, mocker, inventory_update, private_data_dir): + task = tasks.RunInventoryUpdate() + inventory_update.source = 'custom' + inventory_update.source_vars = '{"FOO": "BAR"}' + inventory_update.source_script= CustomInventoryScript(script='#!/bin/sh\necho "Hello, World!"') if with_credential: azure_rm = CredentialType.defaults['azure_rm']() @@ -1833,30 +1791,35 @@ def get_cred(): } ) return cred - self.instance.get_cloud_credential = get_cred + inventory_update.get_cloud_credential = get_cred else: - self.instance.get_cloud_credential = mocker.Mock(return_value=None) - - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - assert '--custom' in ' '.join(args) - script = args[args.index('--source') + 1] - with open(script, 'r') as f: - assert f.read() == self.instance.source_script.script - assert env['FOO'] == 'BAR' - if with_credential: - assert env['AZURE_CLIENT_ID'] == 'some-client' - assert env['AZURE_SECRET'] == 'some-secret' - assert env['AZURE_TENANT'] == 'some-tenant' - assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription' - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + inventory_update.get_cloud_credential = mocker.Mock(return_value=None) + + env = task.build_env(inventory_update, private_data_dir, False) + args = task.build_args(inventory_update, private_data_dir, {}) + + credentials = task.build_credentials_list(inventory_update) + for credential in credentials: + if credential: + credential.credential_type.inject_credential( + credential, env, {}, [], private_data_dir + ) - def test_ec2_source(self): + assert '--custom' in ' '.join(args) + script = args[args.index('--source') + 1] + with open(script, 'r') as f: + assert f.read() == inventory_update.source_script.script + assert env['FOO'] == 'BAR' + if with_credential: + assert env['AZURE_CLIENT_ID'] == 'some-client' + assert env['AZURE_SECRET'] == 'some-secret' + assert env['AZURE_TENANT'] == 'some-tenant' + assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription' + + def test_ec2_source(self, private_data_dir, inventory_update): + task = tasks.RunInventoryUpdate() aws = CredentialType.defaults['aws']() - self.instance.source = 'ec2' + inventory_update.source = 'ec2' def get_cred(): cred = Credential( @@ -1866,27 +1829,33 @@ def get_cred(): ) cred.inputs['password'] = encrypt_field(cred, 'password') return cred - self.instance.get_cloud_credential = get_cred + inventory_update.get_cloud_credential = get_cred - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args + private_data_files = task.build_private_data_files(inventory_update, private_data_dir) + env = task.build_env(inventory_update, private_data_dir, False, private_data_files) - assert env['AWS_ACCESS_KEY_ID'] == 'bob' - assert env['AWS_SECRET_ACCESS_KEY'] == 'secret' - assert 'EC2_INI_PATH' in env + safe_env = {} + credentials = task.build_credentials_list(inventory_update) + for credential in credentials: + if credential: + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) - config = configparser.ConfigParser() - config.read(env['EC2_INI_PATH']) - assert 'ec2' in config.sections() - return ['successful', 0] + assert env['AWS_ACCESS_KEY_ID'] == 'bob' + assert env['AWS_SECRET_ACCESS_KEY'] == 'secret' + assert 'EC2_INI_PATH' in env - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) - assert self.instance.job_env['AWS_SECRET_ACCESS_KEY'] == tasks.HIDDEN_PASSWORD + config = configparser.ConfigParser() + config.read(env['EC2_INI_PATH']) + assert 'ec2' in config.sections() + + assert safe_env['AWS_SECRET_ACCESS_KEY'] == tasks.HIDDEN_PASSWORD - def test_vmware_source(self): + def test_vmware_source(self, inventory_update, private_data_dir): + task = tasks.RunInventoryUpdate() vmware = CredentialType.defaults['vmware']() - self.instance.source = 'vmware' + inventory_update.source = 'vmware' def get_cred(): cred = Credential( @@ -1896,25 +1865,30 @@ def get_cred(): ) cred.inputs['password'] = encrypt_field(cred, 'password') return cred - self.instance.get_cloud_credential = get_cred + inventory_update.get_cloud_credential = get_cred - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args + private_data_files = task.build_private_data_files(inventory_update, private_data_dir) + env = task.build_env(inventory_update, private_data_dir, False, private_data_files) - config = configparser.ConfigParser() - config.read(env['VMWARE_INI_PATH']) - assert config.get('vmware', 'username') == 'bob' - assert config.get('vmware', 'password') == 'secret' - assert config.get('vmware', 'server') == 'https://example.org' - return ['successful', 0] + safe_env = {} + credentials = task.build_credentials_list(inventory_update) + for credential in credentials: + if credential: + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + config = configparser.ConfigParser() + config.read(env['VMWARE_INI_PATH']) + assert config.get('vmware', 'username') == 'bob' + assert config.get('vmware', 'password') == 'secret' + assert config.get('vmware', 'server') == 'https://example.org' - def test_azure_rm_source_with_tenant(self): + def test_azure_rm_source_with_tenant(self, private_data_dir, inventory_update): + task = tasks.RunInventoryUpdate() azure_rm = CredentialType.defaults['azure_rm']() - self.instance.source = 'azure_rm' - self.instance.source_regions = 'north, south, east, west' + inventory_update.source = 'azure_rm' + inventory_update.source_regions = 'north, south, east, west' def get_cred(): cred = Credential( @@ -1929,38 +1903,45 @@ def get_cred(): } ) return cred - self.instance.get_cloud_credential = get_cred - self.instance.source_vars = { + inventory_update.get_cloud_credential = get_cred + inventory_update.source_vars = { 'include_powerstate': 'yes', 'group_by_resource_group': 'no' } - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - assert env['AZURE_CLIENT_ID'] == 'some-client' - assert env['AZURE_SECRET'] == 'some-secret' - assert env['AZURE_TENANT'] == 'some-tenant' - assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription' - assert env['AZURE_CLOUD_ENVIRONMENT'] == 'foobar' + private_data_files = task.build_private_data_files(inventory_update, private_data_dir) + env = task.build_env(inventory_update, private_data_dir, False, private_data_files) - config = configparser.ConfigParser() - config.read(env['AZURE_INI_PATH']) - assert config.get('azure', 'include_powerstate') == 'yes' - assert config.get('azure', 'group_by_resource_group') == 'no' - assert config.get('azure', 'group_by_location') == 'yes' - assert 'group_by_security_group' not in config.items('azure') - assert config.get('azure', 'group_by_tag') == 'yes' - assert config.get('azure', 'locations') == 'north,south,east,west' - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) - assert self.instance.job_env['AZURE_SECRET'] == tasks.HIDDEN_PASSWORD + safe_env = {} + credentials = task.build_credentials_list(inventory_update) + for credential in credentials: + if credential: + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) - def test_azure_rm_source_with_password(self): + assert env['AZURE_CLIENT_ID'] == 'some-client' + assert env['AZURE_SECRET'] == 'some-secret' + assert env['AZURE_TENANT'] == 'some-tenant' + assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription' + assert env['AZURE_CLOUD_ENVIRONMENT'] == 'foobar' + + config = configparser.ConfigParser() + config.read(env['AZURE_INI_PATH']) + assert config.get('azure', 'include_powerstate') == 'yes' + assert config.get('azure', 'group_by_resource_group') == 'no' + assert config.get('azure', 'group_by_location') == 'yes' + assert 'group_by_security_group' not in config.items('azure') + assert config.get('azure', 'group_by_tag') == 'yes' + assert config.get('azure', 'locations') == 'north,south,east,west' + + assert safe_env['AZURE_SECRET'] == tasks.HIDDEN_PASSWORD + + def test_azure_rm_source_with_password(self, private_data_dir, inventory_update): + task = tasks.RunInventoryUpdate() azure_rm = CredentialType.defaults['azure_rm']() - self.instance.source = 'azure_rm' - self.instance.source_regions = 'all' + inventory_update.source = 'azure_rm' + inventory_update.source_regions = 'all' def get_cred(): cred = Credential( @@ -1974,38 +1955,44 @@ def get_cred(): } ) return cred - self.instance.get_cloud_credential = get_cred - self.instance.source_vars = { + inventory_update.get_cloud_credential = get_cred + inventory_update.source_vars = { 'include_powerstate': 'yes', 'group_by_resource_group': 'no', 'group_by_security_group': 'no' } - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription' - assert env['AZURE_AD_USER'] == 'bob' - assert env['AZURE_PASSWORD'] == 'secret' - assert env['AZURE_CLOUD_ENVIRONMENT'] == 'foobar' + private_data_files = task.build_private_data_files(inventory_update, private_data_dir) + env = task.build_env(inventory_update, private_data_dir, False, private_data_files) - config = configparser.ConfigParser() - config.read(env['AZURE_INI_PATH']) - assert config.get('azure', 'include_powerstate') == 'yes' - assert config.get('azure', 'group_by_resource_group') == 'no' - assert config.get('azure', 'group_by_location') == 'yes' - assert config.get('azure', 'group_by_security_group') == 'no' - assert config.get('azure', 'group_by_tag') == 'yes' - assert 'locations' not in config.items('azure') - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) - assert self.instance.job_env['AZURE_PASSWORD'] == tasks.HIDDEN_PASSWORD + safe_env = {} + credentials = task.build_credentials_list(inventory_update) + for credential in credentials: + if credential: + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) + + assert env['AZURE_SUBSCRIPTION_ID'] == 'some-subscription' + assert env['AZURE_AD_USER'] == 'bob' + assert env['AZURE_PASSWORD'] == 'secret' + assert env['AZURE_CLOUD_ENVIRONMENT'] == 'foobar' - def test_gce_source(self): + config = configparser.ConfigParser() + config.read(env['AZURE_INI_PATH']) + assert config.get('azure', 'include_powerstate') == 'yes' + assert config.get('azure', 'group_by_resource_group') == 'no' + assert config.get('azure', 'group_by_location') == 'yes' + assert config.get('azure', 'group_by_security_group') == 'no' + assert config.get('azure', 'group_by_tag') == 'yes' + assert 'locations' not in config.items('azure') + assert safe_env['AZURE_PASSWORD'] == tasks.HIDDEN_PASSWORD + + def test_gce_source(self, inventory_update, private_data_dir): + task = tasks.RunInventoryUpdate() gce = CredentialType.defaults['gce']() - self.instance.source = 'gce' - self.instance.source_regions = 'all' + inventory_update.source = 'gce' + inventory_update.source_regions = 'all' def get_cred(): cred = Credential( @@ -2021,12 +2008,19 @@ def get_cred(): cred, 'ssh_key_data' ) return cred - self.instance.get_cloud_credential = get_cred + inventory_update.get_cloud_credential = get_cred + + def run(expected_gce_zone): + private_data_files = task.build_private_data_files(inventory_update, private_data_dir) + env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + safe_env = {} + credentials = task.build_credentials_list(inventory_update) + for credential in credentials: + if credential: + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) - expected_gce_zone = '' - - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args assert env['GCE_ZONE'] == expected_gce_zone json_data = json.load(open(env['GCE_CREDENTIALS_FILE_PATH'], 'rb')) assert json_data['type'] == 'service_account' @@ -2039,18 +2033,16 @@ def run_pexpect_side_effect(*args, **kwargs): assert 'cache' in config.sections() assert config.getint('cache', 'cache_max_age') == 0 - return ['successful', 0] - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + run('') - self.instance.source_regions = 'us-east-4' - expected_gce_zone = 'us-east-4' - self.task.run(self.pk) + inventory_update.source_regions = 'us-east-4' + run('us-east-4') - def test_openstack_source(self): + def test_openstack_source(self, inventory_update, private_data_dir): + task = tasks.RunInventoryUpdate() openstack = CredentialType.defaults['openstack']() - self.instance.source = 'openstack' + inventory_update.source = 'openstack' def get_cred(): cred = Credential( @@ -2068,29 +2060,27 @@ def get_cred(): cred, 'ssh_key_data' ) return cred - self.instance.get_cloud_credential = get_cred - - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - shade_config = open(env['OS_CLIENT_CONFIG_FILE'], 'r').read() - assert '\n'.join([ - 'clouds:', - ' devstack:', - ' auth:', - ' auth_url: https://keystone.example.org', - ' password: secret', - ' project_name: tenant-name', - ' username: bob', - '' - ]) in shade_config - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) - - def test_satellite6_source(self): + inventory_update.get_cloud_credential = get_cred + + private_data_files = task.build_private_data_files(inventory_update, private_data_dir) + env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + + shade_config = open(env['OS_CLIENT_CONFIG_FILE'], 'r').read() + assert '\n'.join([ + 'clouds:', + ' devstack:', + ' auth:', + ' auth_url: https://keystone.example.org', + ' password: secret', + ' project_name: tenant-name', + ' username: bob', + '' + ]) in shade_config + + def test_satellite6_source(self, inventory_update, private_data_dir): + task = tasks.RunInventoryUpdate() satellite6 = CredentialType.defaults['satellite6']() - self.instance.source = 'satellite6' + inventory_update.source = 'satellite6' def get_cred(): cred = Credential( @@ -2106,28 +2096,26 @@ def get_cred(): cred, 'password' ) return cred - self.instance.get_cloud_credential = get_cred + inventory_update.get_cloud_credential = get_cred - self.instance.source_vars = '{"satellite6_group_patterns": "[a,b,c]", "satellite6_group_prefix": "hey_", "satellite6_want_hostcollections": True}' + inventory_update.source_vars = '{"satellite6_group_patterns": "[a,b,c]", "satellite6_group_prefix": "hey_", "satellite6_want_hostcollections": True}' - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - config = configparser.ConfigParser() - config.read(env['FOREMAN_INI_PATH']) - assert config.get('foreman', 'url') == 'https://example.org' - assert config.get('foreman', 'user') == 'bob' - assert config.get('foreman', 'password') == 'secret' - assert config.get('ansible', 'group_patterns') == '[a,b,c]' - assert config.get('ansible', 'group_prefix') == 'hey_' - assert config.get('ansible', 'want_hostcollections') == 'True' - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + private_data_files = task.build_private_data_files(inventory_update, private_data_dir) + env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + + config = configparser.ConfigParser() + config.read(env['FOREMAN_INI_PATH']) + assert config.get('foreman', 'url') == 'https://example.org' + assert config.get('foreman', 'user') == 'bob' + assert config.get('foreman', 'password') == 'secret' + assert config.get('ansible', 'group_patterns') == '[a,b,c]' + assert config.get('ansible', 'group_prefix') == 'hey_' + assert config.get('ansible', 'want_hostcollections') == 'True' - def test_cloudforms_source(self): + def test_cloudforms_source(self, inventory_update, private_data_dir): + task = tasks.RunInventoryUpdate() cloudforms = CredentialType.defaults['cloudforms']() - self.instance.source = 'cloudforms' + inventory_update.source = 'cloudforms' def get_cred(): cred = Credential( @@ -2143,33 +2131,31 @@ def get_cred(): cred, 'password' ) return cred - self.instance.get_cloud_credential = get_cred + inventory_update.get_cloud_credential = get_cred - self.instance.source_vars = '{"prefer_ipv4": True}' + inventory_update.source_vars = '{"prefer_ipv4": True}' - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - config = configparser.ConfigParser() - config.read(env['CLOUDFORMS_INI_PATH']) - assert config.get('cloudforms', 'url') == 'https://example.org' - assert config.get('cloudforms', 'username') == 'bob' - assert config.get('cloudforms', 'password') == 'secret' - assert config.get('cloudforms', 'ssl_verify') == 'false' - assert config.get('cloudforms', 'prefer_ipv4') == 'True' - - cache_path = config.get('cache', 'path') - assert cache_path.startswith(env['AWX_PRIVATE_DATA_DIR']) - assert os.path.isdir(cache_path) - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) + private_data_files = task.build_private_data_files(inventory_update, private_data_dir) + env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + + config = configparser.ConfigParser() + config.read(env['CLOUDFORMS_INI_PATH']) + assert config.get('cloudforms', 'url') == 'https://example.org' + assert config.get('cloudforms', 'username') == 'bob' + assert config.get('cloudforms', 'password') == 'secret' + assert config.get('cloudforms', 'ssl_verify') == 'false' + assert config.get('cloudforms', 'prefer_ipv4') == 'True' + + cache_path = config.get('cache', 'path') + assert cache_path.startswith(env['AWX_PRIVATE_DATA_DIR']) + assert os.path.isdir(cache_path) @pytest.mark.parametrize('verify', [True, False]) - def test_tower_source(self, verify): + def test_tower_source(self, verify, inventory_update, private_data_dir): + task = tasks.RunInventoryUpdate() tower = CredentialType.defaults['tower']() - self.instance.source = 'tower' - self.instance.instance_filters = '12345' + inventory_update.source = 'tower' + inventory_update.instance_filters = '12345' inputs = { 'host': 'https://tower.example.org', 'username': 'bob', @@ -2181,28 +2167,33 @@ def get_cred(): cred = Credential(pk=1, credential_type=tower, inputs = inputs) cred.inputs['password'] = encrypt_field(cred, 'password') return cred - self.instance.get_cloud_credential = get_cred - - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - assert env['TOWER_HOST'] == 'https://tower.example.org' - assert env['TOWER_USERNAME'] == 'bob' - assert env['TOWER_PASSWORD'] == 'secret' - assert env['TOWER_INVENTORY'] == '12345' - if verify: - assert env['TOWER_VERIFY_SSL'] == 'True' - else: - assert env['TOWER_VERIFY_SSL'] == 'False' - return ['successful', 0] - - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) - assert self.instance.job_env['TOWER_PASSWORD'] == tasks.HIDDEN_PASSWORD + inventory_update.get_cloud_credential = get_cred + + env = task.build_env(inventory_update, private_data_dir, False) - def test_tower_source_ssl_verify_empty(self): + safe_env = {} + credentials = task.build_credentials_list(inventory_update) + for credential in credentials: + if credential: + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) + + assert env['TOWER_HOST'] == 'https://tower.example.org' + assert env['TOWER_USERNAME'] == 'bob' + assert env['TOWER_PASSWORD'] == 'secret' + assert env['TOWER_INVENTORY'] == '12345' + if verify: + assert env['TOWER_VERIFY_SSL'] == 'True' + else: + assert env['TOWER_VERIFY_SSL'] == 'False' + assert safe_env['TOWER_PASSWORD'] == tasks.HIDDEN_PASSWORD + + def test_tower_source_ssl_verify_empty(self, inventory_update, private_data_dir): + task = tasks.RunInventoryUpdate() tower = CredentialType.defaults['tower']() - self.instance.source = 'tower' - self.instance.instance_filters = '12345' + inventory_update.source = 'tower' + inventory_update.instance_filters = '12345' inputs = { 'host': 'https://tower.example.org', 'username': 'bob', @@ -2213,19 +2204,23 @@ def get_cred(): cred = Credential(pk=1, credential_type=tower, inputs = inputs) cred.inputs['password'] = encrypt_field(cred, 'password') return cred - self.instance.get_cloud_credential = get_cred + inventory_update.get_cloud_credential = get_cred + + env = task.build_env(inventory_update, private_data_dir, False) + safe_env = {} + credentials = task.build_credentials_list(inventory_update) + for credential in credentials: + if credential: + credential.credential_type.inject_credential( + credential, env, safe_env, [], private_data_dir + ) - def run_pexpect_side_effect(*args, **kwargs): - args, cwd, env, stdout = args - assert env['TOWER_VERIFY_SSL'] == 'False' - return ['successful', 0] + assert env['TOWER_VERIFY_SSL'] == 'False' - self.run_pexpect.side_effect = run_pexpect_side_effect - self.task.run(self.pk) - - def test_awx_task_env(self): + def test_awx_task_env(self, inventory_update, private_data_dir, settings): + task = tasks.RunInventoryUpdate() gce = CredentialType.defaults['gce']() - self.instance.source = 'gce' + inventory_update.source = 'gce' def get_cred(): cred = Credential( @@ -2237,18 +2232,14 @@ def get_cred(): } ) return cred - self.instance.get_cloud_credential = get_cred + inventory_update.get_cloud_credential = get_cred + settings.AWX_TASK_ENV = {'FOO': 'BAR'} - with mock.patch('awx.main.tasks.settings.AWX_TASK_ENV', {'FOO': 'BAR'}): - self.task.run(self.pk) + env = task.build_env(inventory_update, private_data_dir, False) - assert self.run_pexpect.call_count == 1 - call_args, _ = self.run_pexpect.call_args_list[0] - args, cwd, env, stdout = call_args assert env['FOO'] == 'BAR' - def test_os_open_oserror(): with pytest.raises(OSError): os.open('this_file_does_not_exist', os.O_RDONLY) diff --git a/docs/ansible_runner_integration.md b/docs/ansible_runner_integration.md new file mode 100644 index 000000000000..a27173020ba7 --- /dev/null +++ b/docs/ansible_runner_integration.md @@ -0,0 +1,19 @@ +## Ansible Runner Integration Overview + +Much of the code in AWX around ansible and ansible-playbook invocation interacting has been removed and put into the project ansible-runner. AWX now calls out to ansible-runner to invoke ansible and ansible-playbook. + +### Lifecycle + +In AWX, a task of a certain job type is kicked off (i.e. RunJob, RunProjectUpdate, RunInventoryUpdate, etc) in tasks.py. A temp directory is build to house ansible-runner parameters (i.e. envvars, cmdline, extravars, etc.). The temp directory is filled with the various concepts in AWX (i.e. ssh keys, extra varsk, etc.). The code then builds a set of parameters to be passed to the ansible-runner python module interface, `ansible-runner.interface.run()`. This is where AWX passes control to ansible-runner. Feedback is gathered by AWX via callbacks and handlers passed in. + +The callbacks and handlers are: +* event_handler: Called each time a new event is created in ansible runner. AWX will disptach the event to rabbitmq to be processed on the other end by the callback receiver. +* cancel_callback: Called periodically by ansible runner. This is so that AWX can inform ansible runner if the job should be canceled or not. +* finished_callback: Called once by ansible-runner to denote that the process that was asked to run is finished. AWX will construct the special control event, `EOF`, with an associated total number of events that it observed. +* status_handler: Called by ansible-runner as the process transitions state internally. AWX uses the `starting` status to know that ansible-runner has made all of its decisions around the process that it will launch. AWX gathers and associates these decisions with the Job for historical observation. + +### Debugging + +If you want to debug ansible-runner then set `AWX_CLEANUP_PATHS=False`, run a job, observe the job's `AWX_PRIVATE_DATA_DIR` property, and go the node where the job was executed and inspect that directory. + +If you want to debug the process that ansible-runner invoked (i.e. ansible or ansible-playbook) then observe the job's job_env, job_cwd, and job_args parameters. diff --git a/docs/process_isolation.md b/docs/process_isolation.md index e6e8bd4619de..5afbbd04fa97 100644 --- a/docs/process_isolation.md +++ b/docs/process_isolation.md @@ -4,6 +4,8 @@ In older version of Ansible Tower we used a system called `proot` to isolate tow For Tower 3.1 and later we have switched to using `bubblewrap` which is a much lighter weight and maintained process isolation system. +Tower 3.4 forward uses the process isolation feature in ansible runner to achieve process isolation. + ### Activating Process Isolation By default `bubblewrap` is enabled, this can be turned off via Tower Config or from a tower settings file: