rename to slicing and schema tweaks

awx/api/serializers.py

@@ -3008,7 +3008,7 @@ class Meta:
         fields = ('*', 'host_config_key', 'ask_diff_mode_on_launch', 'ask_variables_on_launch', 'ask_limit_on_launch', 'ask_tags_on_launch',
                   'ask_skip_tags_on_launch', 'ask_job_type_on_launch', 'ask_verbosity_on_launch', 'ask_inventory_on_launch',
                   'ask_credential_on_launch', 'survey_enabled', 'become_enabled', 'diff_mode',
-                  'allow_simultaneous', 'custom_virtualenv', 'job_split_count')
+                  'allow_simultaneous', 'custom_virtualenv', 'job_slice_count')
 
     def get_related(self, obj):
         res = super(JobTemplateSerializer, self).get_related(obj)
@@ -3025,7 +3025,7 @@ def get_related(self, obj):
             labels = self.reverse('api:job_template_label_list', kwargs={'pk': obj.pk}),
             object_roles = self.reverse('api:job_template_object_roles_list', kwargs={'pk': obj.pk}),
             instance_groups = self.reverse('api:job_template_instance_groups_list', kwargs={'pk': obj.pk}),
-            split_jobs = self.reverse('api:job_template_split_jobs_list', kwargs={'pk': obj.pk}),
+            slice_workflow_jobs = self.reverse('api:job_template_slice_workflow_jobs_list', kwargs={'pk': obj.pk}),
         ))
         if self.version > 1:
             res['copy'] = self.reverse('api:job_template_copy', kwargs={'pk': obj.pk})
@@ -3121,7 +3121,7 @@ class Meta:
                   'ask_variables_on_launch', 'ask_limit_on_launch', 'ask_tags_on_launch', 'ask_skip_tags_on_launch',
                   'ask_job_type_on_launch', 'ask_verbosity_on_launch', 'ask_inventory_on_launch',
                   'ask_credential_on_launch', 'allow_simultaneous', 'artifacts', 'scm_revision',
-                  'instance_group', 'diff_mode')
+                  'instance_group', 'diff_mode', 'job_slice_number', 'job_slice_count')
 
     def get_related(self, obj):
         res = super(JobSerializer, self).get_related(obj)
@@ -3199,13 +3199,6 @@ def to_representation(self, obj):
 
     def get_summary_fields(self, obj):
         summary_fields = super(JobSerializer, self).get_summary_fields(obj)
-        if obj.internal_limit:
-            summary_fields['internal_limit'] = {}
-            if obj.internal_limit.startswith('split'):
-                offset, step = Inventory.parse_split_params(obj.internal_limit)
-                summary_fields['internal_limit']['split'] = {'offset': offset, 'step': step}
-            else:
-                summary_fields['internal_limit']['unknown'] = self.internal_limit
         all_creds = []
         # Organize credential data into multitude of deprecated fields
         # TODO: remove most of this as v1 is removed

awx/api/templates/api/inventory_script_view.md

@@ -26,8 +26,8 @@ string of `?all=1` to return all hosts, including disabled ones.
 Specify a query string of `?towervars=1` to add variables
 to the hostvars of each host that specifies its enabled state and database ID.
 
-Specify a query string of `?subset=split2of5` to produce an inventory that
-has a restricted number of hosts according to the rules of job splitting.
+Specify a query string of `?subset=slice2of5` to produce an inventory that
+has a restricted number of hosts according to the rules of job slicing.
 
 To apply multiple query strings, join them with the `&` character, like `?hostvars=1&all=1`.
 

awx/api/urls/job_template.py

@@ -8,7 +8,7 @@
     JobTemplateDetail,
     JobTemplateLaunch,
     JobTemplateJobsList,
-    JobTemplateSplitJobsList,
+    JobTemplateSliceWorkflowJobsList,
     JobTemplateCallback,
     JobTemplateSchedulesList,
     JobTemplateSurveySpec,
@@ -29,7 +29,7 @@
     url(r'^(?P<pk>[0-9]+)/$', JobTemplateDetail.as_view(), name='job_template_detail'),
     url(r'^(?P<pk>[0-9]+)/launch/$', JobTemplateLaunch.as_view(), name='job_template_launch'),
     url(r'^(?P<pk>[0-9]+)/jobs/$', JobTemplateJobsList.as_view(), name='job_template_jobs_list'),
-    url(r'^(?P<pk>[0-9]+)/split_jobs/$', JobTemplateSplitJobsList.as_view(), name='job_template_split_jobs_list'),
+    url(r'^(?P<pk>[0-9]+)/slice_workflow_jobs/$', JobTemplateSliceWorkflowJobsList.as_view(), name='job_template_slice_workflow_jobs_list'),
     url(r'^(?P<pk>[0-9]+)/callback/$', JobTemplateCallback.as_view(), name='job_template_callback'),
     url(r'^(?P<pk>[0-9]+)/schedules/$', JobTemplateSchedulesList.as_view(), name='job_template_schedules_list'),
     url(r'^(?P<pk>[0-9]+)/survey_spec/$', JobTemplateSurveySpec.as_view(), name='job_template_survey_spec'),

awx/api/views/__init__.py

@@ -2453,6 +2453,15 @@ def retrieve(self, request, *args, **kwargs):
         towervars = bool(request.query_params.get('towervars', ''))
         show_all = bool(request.query_params.get('all', ''))
         subset = request.query_params.get('subset', '')
+        if subset:
+            if not isinstance(subset, six.string_types):
+                raise ParseError(_('Inventory subset argument must be a string.'))
+            if subset.startswith('slice'):
+                slice_number, slice_count = Inventory.parse_slice_params(subset)
+            else:
+                raise ParseError(_('Subset does not use any supported syntax.'))
+        else:
+            slice_number, slice_count = 1, 1
         if hostname:
             hosts_q = dict(name=hostname)
             if not show_all:
@@ -2463,7 +2472,7 @@ def retrieve(self, request, *args, **kwargs):
             hostvars=hostvars,
             towervars=towervars,
             show_all=show_all,
-            subset=subset
+            slice_number=slice_number, slice_count=slice_count
         ))
 
 
@@ -3369,7 +3378,7 @@ def post(self, request, *args, **kwargs):
         if extra_vars is not None and job_template.ask_variables_on_launch:
             extra_vars_redacted, removed = extract_ansible_vars(extra_vars)
             kv['extra_vars'] = extra_vars_redacted
-        kv['_prevent_splitting'] = True  # will only run against 1 host, so no point
+        kv['_prevent_slicing'] = True  # will only run against 1 host, so no point
         with transaction.atomic():
             job = job_template.create_job(**kv)
 
@@ -3401,12 +3410,12 @@ def allowed_methods(self):
         return methods
 
 
-class JobTemplateSplitJobsList(SubListCreateAPIView):
+class JobTemplateSliceWorkflowJobsList(SubListCreateAPIView):
 
     model = WorkflowJob
     serializer_class = WorkflowJobListSerializer
     parent_model = JobTemplate
-    relationship = 'split_jobs'
+    relationship = 'slice_workflow_jobs'
     parent_key = 'job_template'
 
 
@@ -3702,6 +3711,8 @@ def get(self, request, *args, **kwargs):
 
     def post(self, request, *args, **kwargs):
         obj = self.get_object()
+        if obj.is_sliced_job and not obj.job_template_id:
+            raise ParseError(_('Cannot relaunch slice workflow job orphaned from job template.'))
         new_workflow_job = obj.create_relaunch_workflow_job()
         new_workflow_job.signal_start()
 

awx/main/migrations/0050_v330_job_slicing.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.11 on 2018-10-15 16:21
+from __future__ import unicode_literals
+
+import awx.main.utils.polymorphic
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0049_v330_validate_instance_capacity_adjustment'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='job',
+            name='job_slice_count',
+            field=models.PositiveIntegerField(blank=True, default=1, help_text='If ran as part of sliced jobs, the total number of slices. If 1, job is not part of a sliced job.'),
+        ),
+        migrations.AddField(
+            model_name='job',
+            name='job_slice_number',
+            field=models.PositiveIntegerField(blank=True, default=0, help_text='If part of a sliced job, the ID of the inventory slice operated on. If not part of sliced job, parameter is not used.'),
+        ),
+        migrations.AddField(
+            model_name='jobtemplate',
+            name='job_slice_count',
+            field=models.PositiveIntegerField(blank=True, default=1, help_text='The number of jobs to slice into at runtime. Will cause the Job Template to launch a workflow if value is greater than 1.'),
+        ),
+        migrations.AddField(
+            model_name='workflowjob',
+            name='is_sliced_job',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='workflowjob',
+            name='job_template',
+            field=models.ForeignKey(blank=True, default=None, help_text='If automatically created for a sliced job run, the job template the workflow job was created from.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='slice_workflow_jobs', to='main.JobTemplate'),
+        ),
+        migrations.AlterField(
+            model_name='unifiedjob',
+            name='unified_job_template',
+            field=models.ForeignKey(default=None, editable=False, null=True, on_delete=awx.main.utils.polymorphic.SET_NULL, related_name='unifiedjob_unified_jobs', to='main.UnifiedJobTemplate'),
+        ),
+    ]

awx/main/models/inventory.py

@@ -221,32 +221,29 @@ def get_group_children_map(self):
         return group_children_map
 
     @staticmethod
-    def parse_split_params(split_str):
-        m = re.match(r"split(?P<offset>\d+)of(?P<step>\d+)", split_str)
+    def parse_slice_params(slice_str):
+        m = re.match(r"slice(?P<number>\d+)of(?P<step>\d+)", slice_str)
         if not m:
-            raise ParseError(_('Could not parse subset as split specification.'))
-        offset = int(m.group('offset'))
+            raise ParseError(_('Could not parse subset as slice specification.'))
+        number = int(m.group('number'))
         step = int(m.group('step'))
-        if offset > step:
-            raise ParseError(_('Split offset must be greater than total number of splits.'))
-        return (offset, step)
+        if number > step:
+            raise ParseError(_('Slice number must be less than total number of slices.'))
+        elif number < 1:
+            raise ParseError(_('Slice number must be 1 or higher.'))
+        return (number, step)
 
-    def get_script_data(self, hostvars=False, towervars=False, show_all=False, subset=None):
+    def get_script_data(self, hostvars=False, towervars=False, show_all=False, slice_number=1, slice_count=1):
         hosts_kw = dict()
         if not show_all:
             hosts_kw['enabled'] = True
         fetch_fields = ['name', 'id', 'variables']
         if towervars:
             fetch_fields.append('enabled')
         hosts = self.hosts.filter(**hosts_kw).order_by('name').only(*fetch_fields)
-        if subset:
-            if not isinstance(subset, six.string_types):
-                raise ParseError(_('Inventory subset argument must be a string.'))
-            if subset.startswith('split'):
-                offset, step = Inventory.parse_split_params(subset)
-                hosts = hosts[offset::step]
-            else:
-                raise ParseError(_('Subset does not use any supported syntax.'))
+        if slice_count > 1:
+            offset = slice_number - 1
+            hosts = hosts[offset::slice_count]
 
         data = dict()
         all_group = data.setdefault('all', dict())

awx/main/models/jobs.py

@@ -277,11 +277,11 @@ class Meta:
         default=False,
         allows_field='credentials'
     )
-    job_split_count = models.IntegerField(
+    job_slice_count = models.PositiveIntegerField(
         blank=True,
-        default=0,
-        help_text=_("The number of jobs to split into at runtime. "
-                    "Will cause the Job Template to launch a workflow if value is non-zero."),
+        default=1,
+        help_text=_("The number of jobs to slice into at runtime. "
+                    "Will cause the Job Template to launch a workflow if value is greater than 1."),
     )
 
     admin_role = ImplicitRoleField(
@@ -302,7 +302,8 @@ def _get_unified_job_class(cls):
     @classmethod
     def _get_unified_job_field_names(cls):
         return set(f.name for f in JobOptions._meta.fields) | set(
-            ['name', 'description', 'schedule', 'survey_passwords', 'labels', 'credentials', 'internal_limit']
+            ['name', 'description', 'schedule', 'survey_passwords', 'labels', 'credentials',
+             'job_slice_number', 'job_slice_count']
         )
 
     @property
@@ -328,25 +329,27 @@ def create_job(self, **kwargs):
         return self.create_unified_job(**kwargs)
 
     def create_unified_job(self, **kwargs):
-        prevent_splitting = kwargs.pop('_prevent_splitting', False)
-        split_event = bool(self.job_split_count > 1 and (not prevent_splitting))
+        prevent_splitting = kwargs.pop('_prevent_slicing', False)
+        split_event = bool(self.job_slice_count > 1 and (not prevent_splitting))
         if split_event:
             # A Split Job Template will generate a WorkflowJob rather than a Job
             from awx.main.models.workflow import WorkflowJobTemplate, WorkflowJobNode
             kwargs['_unified_job_class'] = WorkflowJobTemplate._get_unified_job_class()
             kwargs['_parent_field_name'] = "job_template"
+            kwargs.setdefault('_eager_fields', {})
+            kwargs['_eager_fields']['is_sliced_job'] = True
         job = super(JobTemplate, self).create_unified_job(**kwargs)
         if split_event:
             try:
                 wj_config = job.launch_config
             except JobLaunchConfig.DoesNotExist:
                 wj_config = JobLaunchConfig()
             actual_inventory = wj_config.inventory if wj_config.inventory else self.inventory
-            for idx in xrange(min(self.job_split_count,
+            for idx in xrange(min(self.job_slice_count,
                                   actual_inventory.hosts.count())):
                 create_kwargs = dict(workflow_job=job,
                                      unified_job_template=self,
-                                     ancestor_artifacts=dict(job_split=idx))
+                                     ancestor_artifacts=dict(job_split=idx + 1))
                 WorkflowJobNode.objects.create(**create_kwargs)
         return job
 
@@ -531,10 +534,17 @@ class Meta:
         on_delete=models.SET_NULL,
         help_text=_('The SCM Refresh task used to make sure the playbooks were available for the job run'),
     )
-    internal_limit = models.CharField(
-        max_length=1024,
-        default='',
-        editable=False,
+    job_slice_number = models.PositiveIntegerField(
+        blank=True,
+        default=0,
+        help_text=_("If part of a sliced job, the ID of the inventory slice operated on. "
+                    "If not part of sliced job, parameter is not used."),
+    )
+    job_slice_count = models.PositiveIntegerField(
+        blank=True,
+        default=1,
+        help_text=_("If ran as part of sliced jobs, the total number of slices. "
+                    "If 1, job is not part of a sliced job."),
     )
 
 
@@ -580,10 +590,11 @@ def event_class(self):
         return JobEvent
 
     def copy_unified_job(self, **new_prompts):
-        new_prompts['_prevent_splitting'] = True
-        if self.internal_limit:
-            new_prompts.setdefault('_eager_fields', {})
-            new_prompts['_eager_fields']['internal_limit'] = self.internal_limit  # oddball, not from JT or prompts
+        # Needed for job slice relaunch consistency, do no re-spawn workflow job
+        # target same slice as original job
+        new_prompts['_prevent_slicing'] = True
+        new_prompts.setdefault('_eager_fields', {})
+        new_prompts['_eager_fields']['job_slice_number'] = self.job_slice_number
         return super(Job, self).copy_unified_job(**new_prompts)
 
     @property