Merge pull request #65 from orangespaceman/lm/prepare-results-file

Adds cronjob to prepare results file for manual entry

phx/results/jobs/weekly/prepare_weekly_results.py

@@ -0,0 +1,20 @@
+from datetime import datetime
+
+from django_extensions.management.jobs import WeeklyJob
+from results.results_processor import ResultsProcessor
+
+
+class Job(WeeklyJob):
+    help = "Prepare results spreadsheet for manual entry."
+
+    def execute(self):
+        [curr_file, prev_file] = ResultsProcessor.fetch_results(2)
+
+        processor = ResultsProcessor()
+        processor.process(curr_file, prev_file)
+
+        filename = datetime.today().strftime(
+            "results/NewResults_%Y-%m-%d.xlsx")
+        processor.save(filename)
+
+        # TODO: email file to results crew

phx/results/results_processor.py

@@ -0,0 +1,63 @@
+from typing import List
+
+from files.models import File
+from openpyxl import load_workbook
+
+
+class ResultsProcessor:
+
+    def __init__(self):
+        self.results = None
+        pass
+
+    @staticmethod
+    def fetch_results(num_results: int) -> List[File]:
+        query_set = File.objects.filter(
+            file__contains="BrightonPhoenix",
+            file__iendswith=".xlsx").order_by("-created_date")[:num_results]
+
+        return list(query_set)
+
+    def process(self, curr_file: File, prev_file: File):
+        curr_workbook = load_workbook(filename=curr_file.file.path)
+        prev_workbook = load_workbook(filename=prev_file.file.path)
+
+        curr_results = curr_workbook["Results (30 days)"]
+        prev_results = prev_workbook["Results (30 days)"]
+
+        sheet = curr_workbook.create_sheet(title="NEW RESULTS", index=0)
+        self._append_new_rows(curr_results, prev_results, sheet)
+
+        self.results = curr_workbook
+
+    def _append_new_rows(self, curr_results, prev_results, sheet):
+        for row in curr_results.iter_rows(values_only=True):
+            row = self._normalize_row(row)
+            if row[5] == "parkrun":
+                continue
+
+            seen = False
+            for old_row in prev_results.iter_rows(values_only=True, min_row=2):
+                old_row = self._normalize_row(old_row)
+
+                if row == old_row:
+                    seen = True
+                    break
+
+            if not seen:
+                sheet.append(row)
+
+        return sheet
+
+    def _normalize_row(self, row):
+        # Remove whitespace and convert empty cells to None
+
+        row = tuple(c.strip() if isinstance(c, str) else c for c in row)
+        return tuple(None if c == "" else c for c in row)
+
+    def save(self, filename):
+        if self.results is not None:
+            file = File.objects.create(file=filename)
+            self.results.save(file.file.path)
+        else:
+            raise Exception("No results to save")

phx/results/tests/test_results_processor.py

@@ -0,0 +1,184 @@
+from unittest.mock import patch
+from xmlrpc.client import Boolean
+
+from django.test import TestCase
+from faker import Faker
+from files.models import File
+from openpyxl import Workbook
+from results.results_processor import ResultsProcessor
+
+fake = Faker()
+
+
+def fake_results(num_results: int, seed: int = 1981) -> Workbook:
+    fake.seed_instance(seed)
+
+    wb = Workbook()
+    wb.remove(wb[wb.sheetnames[0]])
+    sheet = wb.create_sheet("Results (30 days)")
+
+    sheet.append([
+        "First Name",
+        "Surname",
+        "Sex",
+        "Age Category",
+        "Date",
+        "Distance",
+        "Race",
+        "Location",
+        "Position",
+        "Age Position",
+        "Gender Position",
+        "Best",
+        "Age Grading",
+        "Club Record",
+        "Time",
+    ])
+
+    for _ in range(num_results):
+        sheet.append(fake_result())
+
+    return wb
+
+
+def fake_result(parkrun: Boolean = False) -> tuple:
+    return (
+        fake.first_name(),
+        fake.last_name(),
+        fake.random_element(["M", "F"]),
+        fake.random_element(["U13", "SEN", "V35", "V50", "V60"]),
+        fake.date(),
+        "parkrun"
+        if parkrun else fake.random_element(["5K", "10K", "HM", "M"]),
+        fake.word(),
+        fake.city(),
+        fake.random_int(),
+        fake.random_int(),
+        fake.random_int(),
+        fake.random_element(["PB", None]),
+        fake.random_int(0, 100),
+        fake.random_element(["Yes", None]),
+        fake.time(),
+    )
+
+
+class TestProcessResults(TestCase):
+
+    def test_fetch_results_returns_an_empty_list_when_no_results(self):
+        self.assertEqual(0, len(ResultsProcessor.fetch_results(2)))
+
+    def test_fetch_results_returns_expected_number_of_files(self):
+        File.objects.create(file="BrightonPhoenix_2024-05-05.xlsx")
+        File.objects.create(file="BrightonPhoenix_2024-05-05.xlsx")
+        File.objects.create(file="BrightonPhoenix_2024-05-12.xlsx")
+        self.assertEqual(2, len(ResultsProcessor.fetch_results(2)))
+
+    def test_fetch_results_returns_most_recent_results(self):
+        File.objects.create(file="BrightonPhoenix_2024-05-05.xlsx")
+        second_file = File.objects.create(
+            file="BrightonPhoenix_2024-05-12.xlsx")
+        self.assertEqual([second_file], ResultsProcessor.fetch_results(1))
+
+    def test_fetch_results_ignores_non_results_files(self):
+        File.objects.create(file="not-results.jpg")
+        self.assertListEqual([], ResultsProcessor.fetch_results(1))
+
+    def test_process_creates_new_sheet_with_new_results(self):
+        old_file = File.objects.create(file="BrightonPhoenix_2024-05-05.xlsx")
+        new_file = File.objects.create(file="BrightonPhoenix_2024-05-12.xlsx")
+
+        with patch("results.results_processor.load_workbook") as mock:
+            mock.side_effect = [fake_results(15), fake_results(10)]
+
+            processor = ResultsProcessor()
+            processor.process(new_file, old_file)
+
+            assert processor.results is not None
+            self.assertEqual(2, len(processor.results.sheetnames))
+            self.assertEqual("NEW RESULTS", processor.results.sheetnames[0])
+            self.assertEqual(6,
+                             len(list(processor.results["NEW RESULTS"].rows)))
+
+    def test_process_populates_new_sheet_with_expected_results(self):
+        old_file = File.objects.create(file="BrightonPhoenix_2024-05-05.xlsx")
+        new_file = File.objects.create(file="BrightonPhoenix_2024-05-12.xlsx")
+
+        with patch("results.results_processor.load_workbook") as mock:
+            prev, curr = (fake_results(10), fake_results(10))
+            new_result = fake_result()  # one new result
+            curr["Results (30 days)"].append(new_result)
+
+            mock.side_effect = [curr, prev]
+
+            processor = ResultsProcessor()
+            processor.process(new_file, old_file)
+
+            assert processor.results is not None
+            sheet = processor.results["NEW RESULTS"]
+            self.assertEqual(2, len(list(sheet.rows)))
+            self.assertEqual(new_result, list(sheet.values)[1])
+
+    def test_process_ignores_parkruns(self):
+        old_file = File.objects.create(file="BrightonPhoenix_2024-05-05.xlsx")
+        new_file = File.objects.create(file="BrightonPhoenix_2024-05-12.xlsx")
+
+        with patch("results.results_processor.load_workbook") as mock:
+            prev, curr = (fake_results(10), fake_results(10))
+            new_result = fake_result(parkrun=True)
+            curr["Results (30 days)"].append(new_result)
+
+            mock.side_effect = [curr, prev]
+
+            processor = ResultsProcessor()
+            processor.process(new_file, old_file)
+
+            assert processor.results is not None
+            sheet = processor.results["NEW RESULTS"]
+            self.assertEqual(1, len(list(sheet.rows)))
+
+    def test_process_ignores_whitespace(self):
+        old_file = File.objects.create(file="BrightonPhoenix_2024-05-05.xlsx")
+        new_file = File.objects.create(file="BrightonPhoenix_2024-05-12.xlsx")
+
+        with patch("results.results_processor.load_workbook") as mock:
+            prev, curr = (fake_results(10), fake_results(10))
+            res = fake_result()
+            leading_whitespace = tuple(f" {value}" for value in res)
+            trailing_whitespace = tuple(f"{value} " for value in res)
+
+            prev["Results (30 days)"].append(leading_whitespace)
+            curr["Results (30 days)"].append(trailing_whitespace)
+
+            mock.side_effect = [curr, prev]
+
+            processor = ResultsProcessor()
+            processor.process(new_file, old_file)
+
+            assert processor.results is not None
+            sheet = processor.results["NEW RESULTS"]
+
+            # only the header row, no new results
+            self.assertEqual(1, len(list(sheet.rows)))
+
+    def test_save_creates_new_file(self):
+        old_file = File.objects.create(file="BrightonPhoenix_2024-05-05.xlsx")
+        new_file = File.objects.create(file="BrightonPhoenix_2024-05-12.xlsx")
+
+        with patch("results.results_processor.load_workbook") as mock:
+            mock.side_effect = [fake_results(10), fake_results(10)]
+
+            processor = ResultsProcessor()
+            processor.process(new_file, old_file)
+
+            filename = "new_results.xlsx"
+            with patch("openpyxl.workbook.Workbook.save") as mock_save:
+                processor.save(filename)
+                mock_save.assert_called_once()
+                self.assertTrue(mock_save.call_args[0][0].endswith(filename))
+
+            self.assertIsNotNone(File.objects.get(file=filename))
+
+    def test_save_raises_exception_when_no_results_to_save(self):
+        processor = ResultsProcessor()
+        with self.assertRaises(Exception):
+            processor.save("new_results.xlsx")

requirements/base.in

@@ -9,3 +9,4 @@ easy-thumbnails
 facebook-sdk
 psycopg2-binary
 twitter
+openpyxl

requirements/base.txt

@@ -38,10 +38,14 @@ django-nested-admin==4.0.2
     # via -r base.in
 easy-thumbnails==2.8.5
     # via -r base.in
+et-xmlfile==1.1.0
+    # via openpyxl
 facebook-sdk==3.1.0
     # via -r base.in
 idna==3.7
     # via requests
+openpyxl==3.1.4
+    # via -r base.in
 pillow==10.3.0
     # via easy-thumbnails
 psycopg2-binary==2.9.9

requirements/ci.txt

@@ -49,6 +49,10 @@ django-nested-admin==4.0.2
     # via -r test.txt
 easy-thumbnails==2.8.5
     # via -r test.txt
+et-xmlfile==1.1.0
+    # via
+    #   -r test.txt
+    #   openpyxl
 exceptiongroup==1.2.1
     # via
     #   -r test.txt
@@ -85,6 +89,8 @@ mccabe==0.7.0
     # via
     #   -r test.txt
     #   flake8
+openpyxl==3.1.4
+    # via -r test.txt
 packaging==24.0
     # via
     #   -r test.txt

requirements/local.txt

@@ -60,6 +60,10 @@ django-nested-admin==4.0.2
     # via -r test.txt
 easy-thumbnails==2.8.5
     # via -r test.txt
+et-xmlfile==1.1.0
+    # via
+    #   -r test.txt
+    #   openpyxl
 exceptiongroup==1.2.1
     # via
     #   -r test.txt
@@ -102,6 +106,8 @@ mccabe==0.7.0
     #   flake8
 nodeenv==1.9.1
     # via pre-commit
+openpyxl==3.1.4
+    # via -r test.txt
 packaging==24.0
     # via
     #   -r test.txt

requirements/production.txt

@@ -44,6 +44,10 @@ django-nested-admin==4.0.2
     # via -r base.txt
 easy-thumbnails==2.8.5
     # via -r base.txt
+et-xmlfile==1.1.0
+    # via
+    #   -r base.txt
+    #   openpyxl
 facebook-sdk==3.1.0
     # via -r base.txt
 gunicorn==22.0.0
@@ -52,6 +56,8 @@ idna==3.7
     # via
     #   -r base.txt
     #   requests
+openpyxl==3.1.4
+    # via -r base.txt
 packaging==24.0
     # via gunicorn
 pillow==10.3.0

requirements/test.txt

@@ -47,6 +47,10 @@ django-nested-admin==4.0.2
     # via -r base.txt
 easy-thumbnails==2.8.5
     # via -r base.txt
+et-xmlfile==1.1.0
+    # via
+    #   -r base.txt
+    #   openpyxl
 exceptiongroup==1.2.1
     # via pytest
 facebook-sdk==3.1.0
@@ -71,6 +75,8 @@ isort==5.13.2
     # via -r test.in
 mccabe==0.7.0
     # via flake8
+openpyxl==3.1.4
+    # via -r base.txt
 packaging==24.0
     # via pytest
 pillow==10.3.0