feat: add command to bulk update course verticals (#4548)

course_discovery/apps/tagging/admin.py

@@ -4,7 +4,7 @@
 from django.core.exceptions import PermissionDenied
 from simple_history.admin import SimpleHistoryAdmin
 
-from course_discovery.apps.tagging.models import CourseVertical, SubVertical, Vertical
+from course_discovery.apps.tagging.models import CourseVertical, SubVertical, UpdateCourseVerticalsConfig, Vertical
 
 
 class SubVerticalInline(admin.TabularInline):
@@ -91,3 +91,11 @@ def save_model(self, request, obj, form, change):
         if not (request.user.is_superuser or request.user.groups.filter(name__in=allowed_groups).exists()):
             raise PermissionDenied("You are not authorized to perform this action.")
         super().save_model(request, obj, form, change)
+
+
+@admin.register(UpdateCourseVerticalsConfig)
+class UpdateCourseVerticalsConfigurationAdmin(admin.ModelAdmin):
+    """
+    Admin for UpdateCourseVerticalsConfig model.
+    """
+    list_display = ('id', 'enabled', 'changed_by', 'change_date')

course_discovery/apps/tagging/emails.py

@@ -0,0 +1,29 @@
+from django.conf import settings
+from django.core.mail import EmailMessage
+from django.template.loader import get_template
+
+
+def send_email_for_course_verticals_update(report, to_users):
+    """
+    Send an overall report of an update_course_verticals mgmt command run
+    """
+    success_count = len(report['successes'])
+    failure_count = len(report['failures'])
+    context = {
+        'total_count': success_count + failure_count,
+        'failure_count': failure_count,
+        'success_count': success_count,
+        'failures': report['failures']
+    }
+    html_template = 'email/update_course_verticals.html'
+    template = get_template(html_template)
+    html_content = template.render(context)
+
+    email = EmailMessage(
+        "Update Course Verticals Command Summary",
+        html_content,
+        settings.PUBLISHER_FROM_EMAIL,
+        to_users,
+    )
+    email.content_subtype = "html"
+    email.send()

course_discovery/apps/tagging/management/commands/tests/test_update_course_verticals.py

@@ -0,0 +1,146 @@
+import ddt
+import pytest
+from bs4 import BeautifulSoup
+from django.core import mail
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django.core.management import CommandError, call_command
+from django.test import TestCase
+
+from course_discovery.apps.course_metadata.tests.factories import CourseFactory
+from course_discovery.apps.tagging.models import CourseVertical, SubVertical, Vertical
+from course_discovery.apps.tagging.tests.factories import (
+    CourseVerticalFactory, SubVerticalFactory, UpdateCourseVerticalsConfigFactory, VerticalFactory
+)
+
+
+@ddt.ddt
+class UpdateCourseVerticalsCommandTests(TestCase):
+    """
+    Test suite for update_course_verticals management command.
+    """
+    def setUp(self):
+        super().setUp()
+        self.course1 = CourseFactory()
+        self.course2 = CourseFactory()
+        self.ai_vertical = VerticalFactory(name="AI & Data Science")
+        self.literature_vertical = VerticalFactory(name="Literature")
+        self.python_subvertical = SubVerticalFactory(name="Python", vertical=self.ai_vertical)
+        self.kafka_subvertical = SubVerticalFactory(name="Kafkaesque", vertical=self.literature_vertical)
+
+        self.csv_data = [
+            {"course": self.course1.key, "vertical": "AI & Data Science", "subvertical": "Python"},
+            {"course": self.course2.key, "vertical": "Literature", "subvertical": "Kafkaesque"},
+        ]
+
+    def build_csv(self, rows):
+        csv_file_content = "course,vertical,subvertical\n"
+        for row in rows:
+            row_content = f"{row["course"]},{row["vertical"]},{row["subvertical"]}\n"
+            csv_file_content += row_content
+
+        csv_file = SimpleUploadedFile(
+            name='test.csv',
+            content=csv_file_content.encode('utf-8'),
+            content_type='text/csv'
+        )
+        return csv_file
+
+    def assert_email_content(self, success_count, failure_count, failure_reasons=None):
+        email = mail.outbox[0]
+        soup = BeautifulSoup(email.body)
+
+        table = soup.find('table', {'width': '50%'})
+        rows = table.find_all('tr')
+
+        assert len(rows) == 4
+        assert 'Total' in rows[1].find('th').get_text()
+        assert 'Success' in rows[2].find('th').get_text()
+        assert 'Failure' in rows[3].find('th').get_text()
+        assert f'{success_count + failure_count}' == rows[1].find('td').get_text()
+        assert f'{success_count}' == rows[2].find('td').get_text()
+        assert f'{failure_count}' == rows[3].find('td').get_text()
+
+        if failure_reasons:
+            assert 'Failures' in soup.find('h3').get_text()
+            failures = soup.find_all('li')
+            for i, (key, value) in enumerate(failure_reasons.items()):
+                assert failures[i].get_text().startswith(f"[{key}]: {value}")
+        else:
+            assert soup.find('h3') is None
+
+    @ddt.data(True, False)
+    def test_success(self, has_existing_verticals):
+        if has_existing_verticals:
+            CourseVerticalFactory(
+                course=self.course1, vertical=self.literature_vertical, sub_vertical=self.kafka_subvertical
+            )
+            CourseVerticalFactory(
+                course=self.course2, vertical=self.ai_vertical, sub_vertical=self.python_subvertical
+            )
+            assert CourseVertical.objects.count() == 2
+        else:
+            assert CourseVertical.objects.count() == 0
+
+        csv = self.build_csv(self.csv_data)
+        UpdateCourseVerticalsConfigFactory(enabled=True, csv_file=csv)
+        call_command('update_course_verticals')
+
+        self.course1.refresh_from_db()
+        self.course2.refresh_from_db()
+        assert self.course1.vertical.vertical == self.ai_vertical
+        assert self.course1.vertical.sub_vertical == self.python_subvertical
+        assert self.course2.vertical.vertical == self.literature_vertical
+        assert self.course2.vertical.sub_vertical == self.kafka_subvertical
+        assert CourseVertical.objects.count() == 2
+        assert Vertical.objects.count() == 2
+        assert SubVertical.objects.count() == 2
+        self.assert_email_content(success_count=2, failure_count=0)
+
+    def test_empty_subvertical(self):
+        self.csv_data.pop()
+        self.csv_data[0]['subvertical'] = ''
+        csv = self.build_csv(self.csv_data)
+        UpdateCourseVerticalsConfigFactory(enabled=True, csv_file=csv)
+        call_command('update_course_verticals')
+
+        assert self.course1.vertical.vertical == self.ai_vertical
+        assert self.course1.vertical.sub_vertical is None
+        assert not hasattr(self.course2, 'vertical')
+        assert CourseVertical.objects.count() == 1
+        self.assert_email_content(success_count=1, failure_count=0)
+
+    def test_nonexistent_vertical(self):
+        self.csv_data[0]["vertical"] = "Computer Science"
+        csv = self.build_csv(self.csv_data)
+        UpdateCourseVerticalsConfigFactory(enabled=True, csv_file=csv)
+        call_command('update_course_verticals')
+
+        assert not hasattr(self.course1, 'vertical')
+        assert self.course2.vertical.vertical == self.literature_vertical
+        assert self.course2.vertical.sub_vertical == self.kafka_subvertical
+        assert CourseVertical.objects.count() == 1
+        self.assert_email_content(
+            success_count=1, failure_count=1, failure_reasons={f"{self.course1.key}": "ValueError"}
+        )
+
+    def test_inactive_vertical(self):
+        self.literature_vertical.is_active = False
+        self.literature_vertical.save()
+
+        csv = self.build_csv(self.csv_data)
+        UpdateCourseVerticalsConfigFactory(enabled=True, csv_file=csv)
+        call_command('update_course_verticals')
+
+        assert self.course1.vertical.vertical == self.ai_vertical
+        assert self.course1.vertical.sub_vertical == self.python_subvertical
+        assert not hasattr(self.course2, 'vertical')
+        assert CourseVertical.objects.count() == 1
+        self.assert_email_content(
+            success_count=1, failure_count=1, failure_reasons={f"{self.course2.key}": "ValueError"}
+        )
+
+    def test_raises_error_if_config_disabled(self):
+        with pytest.raises(CommandError):
+            csv = self.build_csv(self.csv_data)
+            UpdateCourseVerticalsConfigFactory(enabled=False, csv_file=csv)
+            call_command("update_course_verticals")

course_discovery/apps/tagging/management/commands/update_course_verticals.py

@@ -0,0 +1,85 @@
+"""
+Management command for updating course verticals and subverticals
+
+Example usage:
+    $ ./manage.py update_course_verticals
+
+"""
+import logging
+
+import unicodecsv
+from django.conf import settings
+from django.core.management import BaseCommand, CommandError
+
+from course_discovery.apps.course_metadata.models import Course
+from course_discovery.apps.tagging.emails import send_email_for_course_verticals_update
+from course_discovery.apps.tagging.models import CourseVertical, SubVertical, UpdateCourseVerticalsConfig, Vertical
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Update course verticals and subverticals"
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.report = {
+            'failures': [],
+            'successes': [],
+        }
+
+    def handle(self, *args, **options):
+        reader = self.get_csv_reader()
+
+        for row in reader:
+            try:
+                course_key = row.get('course')
+                self.process_vertical_information(row)
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                self.report['failures'].append(
+                    {
+                        'id': course_key,  # pylint: disable=used-before-assignment
+                        'reason': repr(exc)
+                    }
+                )
+                logger.exception(f"Failed to set vertical/subvertical information for course with key {course_key}")
+            else:
+                self.report['successes'].append(
+                    {
+                        'id': course_key,
+                    }
+                )
+                logger.info(f"Successfully set vertical and subvertical info for course with key {course_key}")
+
+        send_email_for_course_verticals_update(self.report, settings.COURSE_VERTICALS_UPDATE_RECIPIENTS)
+
+    def process_vertical_information(self, row):
+        course_key, vertical_name, subvertical_name = row['course'], row['vertical'], row['subvertical']
+        course = Course.objects.get(key=course_key)
+        vertical = Vertical.objects.filter(name=vertical_name).first()
+        subvertical = SubVertical.objects.filter(name=subvertical_name).first()
+        if (not vertical and vertical_name) or (not subvertical and subvertical_name):
+            raise ValueError("Incorrect vertical or subvertical provided")
+
+        if (vertical and not vertical.is_active) or (subvertical and not subvertical.is_active):
+            raise ValueError("The provided vertical or subvertical is not active")
+
+        course_vertical = CourseVertical.objects.filter(course=course).first()
+        if course_vertical:
+            logger.info(f"Existing vertical association found for course with key {course_key}")
+            course_vertical.vertical = vertical
+            course_vertical.sub_vertical = subvertical
+            course_vertical.save()
+        else:
+            CourseVertical.objects.create(course=course, vertical=vertical, sub_vertical=subvertical)
+
+    def get_csv_reader(self):
+        config = UpdateCourseVerticalsConfig.current()
+        if not config.enabled:
+            raise CommandError('Configuration object is not enabled')
+
+        if not config.csv_file:
+            raise CommandError('Configuration object does not have any input csv')
+
+        reader = unicodecsv.DictReader(config.csv_file)
+        return reader

course_discovery/apps/tagging/migrations/0002_updatecourseverticalsconfig.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.2.17 on 2025-01-21 12:05
+
+from django.conf import settings
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('tagging', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='UpdateCourseVerticalsConfig',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('change_date', models.DateTimeField(auto_now_add=True, verbose_name='Change date')),
+                ('enabled', models.BooleanField(default=False, verbose_name='Enabled')),
+                ('csv_file', models.FileField(help_text='A csv file containing the course keys, verticals and subverticals', upload_to='', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['csv'])])),
+                ('changed_by', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL, verbose_name='Changed by')),
+            ],
+            options={
+                'ordering': ('-change_date',),
+                'abstract': False,
+            },
+        ),
+    ]

course_discovery/apps/tagging/models.py

@@ -1,4 +1,6 @@
+from config_models.models import ConfigurationModel
 from django.core.exceptions import ValidationError
+from django.core.validators import FileExtensionValidator
 from django.db import models
 from django_extensions.db.fields import AutoSlugField
 from model_utils.models import TimeStampedModel
@@ -111,3 +113,15 @@ def save(self, *args, **kwargs):
 
     def get_object_title(self):
         return self.course.title
+
+
+class UpdateCourseVerticalsConfig(ConfigurationModel):
+    """
+    Configuration to store a csv file for the update_course_verticals command
+    """
+    # Timeout set to 0 so that the model does not read from cached config in case the config entry is deleted.
+    cache_timeout = 0
+    csv_file = models.FileField(
+        validators=[FileExtensionValidator(allowed_extensions=['csv'])],
+        help_text="A csv file containing the course keys, verticals and subverticals"
+    )

course_discovery/apps/tagging/templates/email/update_course_verticals.html

@@ -0,0 +1,42 @@
+{% extends "course_metadata/email/email_base.html" %}
+{% load django_markup %}
+{% block body %}
+
+<p>
+    The course verticals update process has completed. A summary is presented below.
+</p>
+<div>
+    <table border="2" width="50%" style="padding: 5px;">
+        <tr>
+            <th colspan="2">
+                Course Verticals Update Summary
+            </th>
+        </tr>
+        <tr>
+            <th>Total Data Rows</th>
+            <td>{{ total_count }}</td>
+        </tr>
+        <tr>
+            <th>Successfully Updated</th>
+            <td>{{ success_count }}</td>
+        </tr>
+        <tr>
+            <th>Failures</th>
+            <td>{{ failure_count }}</td>
+        </tr>
+    </table>
+</div>
+
+
+{% if failure_count > 0 %}
+<div>
+    <h3>Verticals Update Failures</h3>
+        <ul>
+            {% for failure in failures %}
+                <li>[{{failure.id}}]: {{failure.reason}}</li>
+            {% endfor %}
+        </ul>
+</div>
+{% endif %}
+
+{% endblock body %}