FFT-170 adding integration tests for payroll ingestion

payroll/services/ingest.py

@@ -75,6 +75,8 @@ def import_payroll(payroll_csv: File) -> ImportPayrollReport:
     grades = set(Grade.objects.values_list("pk", flat=True))
 
     for row in csv_reader:
+        if is_row_empty(row):
+            continue
         emp_dict = _csv_row_employee_dict(PayrollRow(*row))
         emp_no = emp_dict["employee_no"]
 
@@ -133,3 +135,7 @@ def import_payroll(payroll_csv: File) -> ImportPayrollReport:
 
 def _csv_row_employee_dict(hr_row) -> EmployeeDict:
     return {x: y(hr_row) for x, y in row_to_employee_dict.items()}
+
+
+def is_row_empty(row):
+    return not any(str(cell).strip() for cell in row)

payroll/templates/payroll/page/import_payroll.html

@@ -20,7 +20,11 @@ <h1 class="govuk-heading-l">Import payroll</h1>
     </form>
     <div>
         <br/><br/>
+
         {% if output %}
+        {% if output.error %}
+        <div>{{output.error}}</div>
+        {% endif %}
             Failing records: {{output.failed|length}} <br/>
             Inserted records : {{output.created}} <br/>
             Updated records: {{output.updated}} <br/>

payroll/tests/services/test_ingest.py

@@ -0,0 +1,77 @@
+from pathlib import Path
+
+import pytest
+from django.core.files import File
+
+from chartofaccountDIT.test.factories import ProgrammeCodeFactory
+from costcentre.test.factories import CostCentreFactory
+from gifthospitality.test.factories import GradeFactory
+from payroll.models import Employee
+
+from ...services.ingest import import_payroll
+
+
+TEST_DATA_DIR = Path(__file__).parent.parent / "test_assets"
+
+
+@pytest.fixture(autouse=True)
+def setup(db):
+    cost_centre_codes = ["888812", "888813", "888814"]
+    programme_codes = ["338887", "338888", "338889"]
+    grade_codes = [
+        "SEO",
+        "SCS",
+        "HEO",
+        "Grade 7",
+        "Grade 6",
+        "Faststream",
+        "EO",
+        "Contractor",
+        "AO",
+        "AA",
+    ]
+    [CostCentreFactory(cost_centre_code=id) for id in cost_centre_codes]
+    [ProgrammeCodeFactory(programme_code=id) for id in programme_codes]
+    [GradeFactory(grade=id) for id in grade_codes]
+    yield
+
+
+def test_ingest_payroll_success(db):
+    """Testing valid records"""
+    csv_file = TEST_DATA_DIR / "payroll_valid_records.csv"
+    with open(csv_file, "rb") as f:
+        result = import_payroll(File(f))
+        assert len(list(Employee.objects.all())) == 20
+        assert len(result.get("failed")) == 0
+        assert result.get("error") is None
+        assert result.get("created") == 20
+
+
+def test_ingest_payroll_failed_record(db):
+    """Testing failed records"""
+    csv_file = TEST_DATA_DIR / "payroll_mixed_records.csv"
+    with open(csv_file, "rb") as f:
+        result = import_payroll(File(f))
+        assert result.get("failed") is not None
+        assert len(list(Employee.objects.all())) == 15
+
+
+def test_ingest_payroll_error(db):
+    """Testing mall structured  csv file"""
+    csv_file = TEST_DATA_DIR / "payroll_empty_rows.csv"
+    with open(csv_file, "rb") as f:
+        import_payroll(File(f))
+        assert len(list(Employee.objects.all())) == 15
+
+
+def test_ingest_payroll_update(db):
+    """Testing update record functionality"""
+    csv_file = TEST_DATA_DIR / "payroll_valid_records.csv"
+    with open(csv_file, "rb") as f:
+        result = import_payroll(File(f))
+        result = import_payroll(File(f))
+        assert len(result.get("failed")) == 0
+        assert result.get("error") is None
+        assert result.get("updated") == 20
+        assert result.get("created") == 0
+        assert len(list(Employee.objects.all())) == 20

payroll/tests/test_assets/payroll_empty_rows.csv

@@ -0,0 +1,24 @@
+employee_no,first_name,last_name,cost_centre_code,programme_code,grade,assignment_status,fte,basic_pay,ernic,pension
+150891,Charlotte,Wilson,1888812,338887,SEO,Secondment In - Unpaid,0,268154,674128,646780
+150892,Jack,Wright,888813,338888,SCS,Loan Out - Non Payroll,0,0,0,0
+150893,Oliver,Thomas,888814,338889,HEO,Secondment In - Unpaid,0,91692,109058,467950
+
+
+
+150894,Lucy,Wilson,888812,338887,Grade 7,Secondment In - Unpaid,0,541545,821337,353586
+150895,Lucy,Brown,888813,338888,Grade 6,Active Assignment,1,225309,951986,988960
+150896,Emily,Taylor,888814,338889,Faststream,Active Assignment,1,768384,552567,321801
+150897,Elizabeth,Brown,888812,338887,EO,Loan In - Payroll,1,823719,706068,254764
+150898,Elizabeth,Williams,888813,338888,Contractor,Loan Out - Non Payroll,0,0,0,0
+150899,George,Williams,888814,338889,AO,Loan In - Payroll,1,827779,759096,925580
+150900,Lucy,Roberts,888812,338887,AA,Loan In - Payroll,1,294191,170873,757744
+150901,Charlie,Brown,888813,338888,SEO,Loan In - Payroll,1,149978,671475,802482
+150902,George,Jones,888814,338889,SCS,Loan In - Payroll,1,641702,475572,480622
+150903,Emma,Taylor,888812,338887,HEO-1,Loan In - Payroll,1,875298,451712,586056
+150904,Lucy,Smith,1888813,338888,Grade 7,Loan In - Payroll,1,130034,946561,673974
+150905,Alice,Davies,888814,338889,Grade 6,Secondment In - Unpaid,0,762957,720411,922423
+150906,Emma,Wilson,1888812,338887,Faststream,Loan In - Payroll,1,445218,840452,580614
+150907,Lucy,Jones,888813,338888,EO,Loan Out - Non Payroll,0,0,0,0
+150908,Oliver,Evans,888814,338889,Contractor,Secondment Out - Paid,1,314034,274579,327408
+150909,Jack,Smithhhh,888814,338887,AO,Loan Out - Non Payroll,0,0,0,0
+150910,George,Davies,888812,338888,Some GRADE,Secondment Out - Paid,1,360808,296336,883975

payroll/tests/test_assets/payroll_mixed_records.csv

@@ -0,0 +1,21 @@
+employee_no,first_name,last_name,cost_centre_code,programme_code,grade,assignment_status,fte,basic_pay,ernic,pension
+150891,Charlotte,Wilson,1888812,338887,SEO,Secondment In - Unpaid,0,268154,674128,646780
+150892,Jack,Wright,888813,338888,SCS,Loan Out - Non Payroll,0,0,0,0
+150893,Oliver,Thomas,888814,338889,HEO,Secondment In - Unpaid,0,91692,109058,467950
+150894,Lucy,Wilson,888812,338887,Grade 7,Secondment In - Unpaid,0,541545,821337,353586
+150895,Lucy,Brown,888813,338888,Grade 6,Active Assignment,1,225309,951986,988960
+150896,Emily,Taylor,888814,338889,Faststream,Active Assignment,1,768384,552567,321801
+150897,Elizabeth,Brown,888812,338887,EO,Loan In - Payroll,1,823719,706068,254764
+150898,Elizabeth,Williams,888813,338888,Contractor,Loan Out - Non Payroll,0,0,0,0
+150899,George,Williams,888814,338889,AO,Loan In - Payroll,1,827779,759096,925580
+150900,Lucy,Roberts,888812,338887,AA,Loan In - Payroll,1,294191,170873,757744
+150901,Charlie,Brown,888813,338888,SEO,Loan In - Payroll,1,149978,671475,802482
+150902,George,Jones,888814,338889,SCS,Loan In - Payroll,1,641702,475572,480622
+150903,Emma,Taylor,888812,338887,HEO-1,Loan In - Payroll,1,875298,451712,586056
+150904,Lucy,Smith,1888813,338888,Grade 7,Loan In - Payroll,1,130034,946561,673974
+150905,Alice,Davies,888814,338889,Grade 6,Secondment In - Unpaid,0,762957,720411,922423
+150906,Emma,Wilson,1888812,338887,Faststream,Loan In - Payroll,1,445218,840452,580614
+150907,Lucy,Jones,888813,338888,EO,Loan Out - Non Payroll,0,0,0,0
+150908,Oliver,Evans,888814,338889,Contractor,Secondment Out - Paid,1,314034,274579,327408
+150909,Jack,Smithhhh,888814,338887,AO,Loan Out - Non Payroll,0,0,0,0
+150910,George,Davies,888812,1338888,Some GRADE,Secondment Out - Paid,1,360808,296336,883975

payroll/tests/test_assets/payroll_valid_records.csv

@@ -0,0 +1,21 @@
+employee_no,first_name,last_name,cost_centre_code,programme_code,grade,assignment_status,fte,basic_pay,ernic,pension
+150892,Jack,Wright,888813,338888,SCS,Loan Out - Non Payroll,0,0,0,0
+150893,Oliver,Thomas,888814,338889,HEO,Secondment In - Unpaid,0,91692,109058,467950
+150894,Lucy,Wilson,888812,338887,Grade 7,Secondment In - Unpaid,0,541545,821337,353586
+150895,Lucy,Brown,888813,338888,Grade 6,Active Assignment,1,225309,951986,988960
+150896,Emily,Taylor,888814,338889,Faststream,Active Assignment,1,768384,552567,321801
+150897,Elizabeth,Brown,888812,338887,EO,Loan In - Payroll,1,823719,706068,254764
+150898,Elizabeth,Williams,888813,338888,Contractor,Loan Out - Non Payroll,0,0,0,0
+150899,George,Williams,888814,338889,AO,Loan In - Payroll,1,827779,759096,925580
+150900,Lucy,Roberts,888812,338887,AA,Loan In - Payroll,1,294191,170873,757744
+150901,Charlie,Brown,888813,338888,SEO,Loan In - Payroll,1,149978,671475,802482
+150902,George,Jones,888814,338889,SCS,Loan In - Payroll,1,641702,475572,480622
+150905,Alice,Davies,888814,338889,Grade 6,Secondment In - Unpaid,0,762957,720411,922423
+150907,Lucy,Jones,888813,338888,EO,Loan Out - Non Payroll,0,0,0,0
+150908,Oliver,Evans,888814,338889,Contractor,Secondment Out - Paid,1,314034,274579,327408
+150909,Jack,Smithhhh,888814,338887,AO,Loan Out - Non Payroll,0,0,0,0
+150891,Charlotte,Wilson,888812,338887,SEO,Secondment In - Unpaid,0,268154,674128,646780
+150903,Emma,Taylor,888812,338887,HEO,Loan In - Payroll,1,875298,451712,586056
+150904,Lucy,Smith,888813,338888,Grade 7,Loan In - Payroll,1,130034,946561,673974
+150906,Emma,Wilson,888812,338887,Faststream,Loan In - Payroll,1,445218,840452,580614
+150910,George,Davies,888812,338888,Faststream,Secondment Out - Paid,1,360808,296336,883975