Merge pull request #531 from tomx4096/dump_dept_csv_489

Dump departmental data as CSV files

OpenOversight/app/main/views.py

@@ -9,7 +9,7 @@
 from werkzeug import secure_filename
 
 from flask import (abort, render_template, request, redirect, url_for,
-                   flash, current_app, jsonify)
+                   flash, current_app, jsonify, Response)
 from flask_login import current_user, login_required, login_user
 
 from . import main
@@ -609,6 +609,83 @@ def submit_data():
         return render_template('submit_image.html', form=form, preferred_dept_id=preferred_dept_id)
 
 
+def check_input(str_input):
+    if str_input is None or str_input == "Not Sure":
+        return ""
+    else:
+        return str(str_input).replace(",", " ")  # no commas allowed
+
+
+@main.route('/download/department/<int:department_id>', methods=['GET'])
+@limiter.limit('5/minute')
+def download_dept_csv(department_id):
+    department = Department.query.filter_by(id=department_id).first()
+    records = Officer.query.filter_by(department_id=department_id).all()
+    if not department or not records:
+        abort(404)
+    dept_name = records[0].department.name.replace(" ", "_")
+    first_row = "id, last, first, middle, suffix, gender, "\
+                "race, born, employment_date, assignments\n"
+
+    assign_dict = {}
+    assign_records = Assignment.query.all()
+    for r in assign_records:
+        if r.officer_id not in assign_dict:
+            assign_dict[r.officer_id] = []
+        assign_dict[r.officer_id].append("(#%s %s %s %s %s)" % (check_input(r.star_no), check_input(r.rank), check_input(r.unit), check_input(r.star_date), check_input(r.resign_date)))
+
+    record_list = ["%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n" %
+                   (str(record.id),
+                    check_input(record.last_name),
+                    check_input(record.first_name),
+                    check_input(record.middle_initial),
+                    check_input(record.suffix),
+                    check_input(record.gender),
+                    check_input(record.race),
+                    check_input(record.birth_year),
+                    check_input(record.employment_date),
+                    " ".join(assign_dict.get(record.id, [])),
+                    ) for record in records]
+
+    csv_name = dept_name + "_Officers.csv"
+    csv = first_row + "".join(record_list)
+    csv_headers = {"Content-disposition": "attachment; filename=" + csv_name}
+    return Response(csv, mimetype="text/csv", headers=csv_headers)
+
+
+@main.route('/download/department/<int:department_id>/incidents', methods=['GET'])
+@limiter.limit('5/minute')
+def download_incidents_csv(department_id):
+    department = Department.query.filter_by(id=department_id).first()
+    records = Incident.query.filter_by(department_id=department.id).all()
+    if not department or not records:
+        abort(404)
+    dept_name = records[0].department.name.replace(" ", "_")
+    first_row = "id,report_num,date,description,location,licences,links,officers\n"
+
+    record_list = ["%s,%s,%s,%s,%s,%s,%s,%s\n" %
+                   (str(record.id),
+                    check_input(record.report_number),
+                    check_input(record.date),
+                    check_input(record.description),
+                    check_input(record.address),
+                    " ".join(map(lambda x: str(x), record.license_plates)),
+                    " ".join(map(lambda x: str(x), record.links)),
+                    " ".join(map(lambda x: str(x), record.officers)),
+                    ) for record in records]
+
+    csv_name = dept_name + "_Incidents.csv"
+    csv = first_row + "".join(record_list)
+    csv_headers = {"Content-disposition": "attachment; filename=" + csv_name}
+    return Response(csv, mimetype="text/csv", headers=csv_headers)
+
+
+@main.route('/download/all', methods=['GET'])
+def all_data():
+    departments = Department.query.all()
+    return render_template('all_depts.html', departments=departments)
+
+
 @main.route('/upload/department/<int:department_id>', methods=['POST'])
 @limiter.limit('250/minute')
 def upload(department_id):

OpenOversight/app/models.py

@@ -237,6 +237,20 @@ def validate_zip_code(self, key, zip_code):
     def validate_state(self, key, state):
         return state_validator(state)
 
+    def __repr__(self):
+        if self.street_name and self.cross_street2:
+            return 'Intersection of {} and {}, {} {}'.format(
+                self.street_name, self.cross_street2, self.city, self.state)
+        elif self.street_name and self.cross_street1:
+            return 'Intersection of {} and {}, {} {}'.format(
+                self.street_name, self.cross_street1, self.city, self.state)
+        elif self.street_name and self.cross_street1 and self.cross_street2:
+            return 'Intersection of {} between {} and {}, {} {}'.format(
+                self.street_name, self.cross_street1, self.cross_street2,
+                self.city, self.state)
+        else:
+            return '{} {}'.format(self.city, self.state)
+
 
 class LicensePlate(db.Model):
     __tablename__ = 'license_plates'

OpenOversight/app/templates/all_depts.html

@@ -0,0 +1,22 @@
+{% extends "base.html" %}
+{% import "bootstrap/wtf.html" as wtf %}
+{% block content %}
+
+<div class="container theme-showcase" role="main">
+
+  <!--div class="text-center frontpage-leads"-->
+  {% for dept in departments %}
+    <p>
+       {{ dept.name }}
+       <a href={{ url_for('main.download_dept_csv',department_id=dept.id) }}>officers.csv</a> 
+
+       {% if dept.incidents %}
+           <a href={{ url_for('main.download_incidents_csv',department_id=dept.id) }}>incidents.csv</a>
+       {% endif %}
+
+    </p>
+  {% endfor %}
+  <!--/div-->
+</div>
+
+{% endblock %}

OpenOversight/app/templates/base.html

@@ -101,7 +101,8 @@ <h5>OpenOversight</h5>
           <p class="font-weight-300">
             A product of Lucy Parsons Labs in Chicago, IL. <br>
             <a href="https://lucyparsonslabs.com/" target="_blank" class="btn-unstyled">Lucy Parsons Labs</a><br>
-            <a href="https://lucyparsonslabs.com/securedrop/" target="_blank" class="btn-unstyled">SecureDrop</a>
+            <a href="https://lucyparsonslabs.com/securedrop/" target="_blank" class="btn-unstyled">SecureDrop</a><br>
+            <a href="{{ url_for('main.all_data') }}" target="_blank" class="btn-unstyled">Download department data</a>
           </p>
         </div>
 

OpenOversight/tests/routes/test_officer_and_department.py

@@ -1,7 +1,7 @@
 # Routing and view tests
 import pytest
 import random
-import datetime
+from datetime import datetime
 from flask import url_for, current_app
 from ..conftest import AC_DEPT
 from OpenOversight.app.utils import dept_choices
@@ -12,7 +12,9 @@
 from OpenOversight.app.main.forms import (AssignmentForm, DepartmentForm,
                                           AddOfficerForm, AddUnitForm,
                                           EditOfficerForm, LinkForm,
-                                          EditDepartmentForm, BrowseForm)
+                                          EditDepartmentForm, IncidentForm,
+                                          LocationForm, LicensePlateForm,
+                                          BrowseForm)
 
 from OpenOversight.app.models import Department, Unit, Officer
 
@@ -873,6 +875,84 @@ def test_admin_can_add_new_officer_with_suffix(mockdata, client, session):
         assert officer.suffix == 'Jr'
 
 
+def test_officer_csv(mockdata, client, session):
+    with current_app.test_request_context():
+        login_admin(client)
+        department = random.choice(dept_choices())
+        links = [
+            LinkForm(url='http://www.pleasework.com', link_type='link').data,
+        ]
+        form = AddOfficerForm(first_name='CKtVwe2gqhAIc',
+                              last_name='FVkcjigWUeUyA',
+                              middle_initial='T',
+                              suffix='Jr',
+                              race='WHITE',
+                              gender='M',
+                              star_no=90009,
+                              rank='PO',
+                              department=department.id,
+                              birth_year=1910,
+                              links=links)
+        # add the officer
+        rv = client.post(
+            url_for('main.add_officer'),
+            data=process_form_data(form.data),
+            follow_redirects=True
+        )
+        # dump officer csv
+        rv = client.get(
+            url_for('main.download_dept_csv', department_id=department.id),
+            follow_redirects=True
+        )
+        # get csv entry matching officer last name
+        csv = filter(lambda row: form.last_name.data in row, rv.data.split("\n"))
+        assert len(csv) == 1
+        assert form.first_name.data in csv[0]
+        assert form.last_name.data in csv[0]
+        assert form.rank.data in csv[0]
+
+
+def test_incidents_csv(mockdata, client, session):
+    with current_app.test_request_context():
+        login_admin(client)
+        department = random.choice(dept_choices())
+        incident_date = datetime(2000, 5, 25, 1, 45)
+        report_number = '42'
+
+        address_form = LocationForm(street_name='ABCDE', city='FGHI', state='IA')
+        link_form = LinkForm(url='http://example.com', link_type='video')
+        license_plates_form = LicensePlateForm(state='AZ')
+        form = IncidentForm(
+            date_field=str(incident_date.date()),
+            time_field=str(incident_date.time()),
+            report_number=report_number,
+            description='Something happened',
+            department=str(department.id),
+            department_id=department.id,
+            address=address_form.data,
+            links=[link_form.data],
+            license_plates=[license_plates_form.data],
+            officers=[]
+        )
+        # add the incident
+        rv = client.post(
+            url_for('main.incident_api') + 'new',
+            data=process_form_data(form.data),
+            follow_redirects=True
+        )
+        assert "created" in rv.data
+        # dump incident csv
+        rv = client.get(
+            url_for('main.download_incidents_csv', department_id=department.id),
+            follow_redirects=True
+        )
+        # print(rv.data)
+        # get the csv entry with matching report number
+        csv = filter(lambda row: report_number in row, rv.data.split("\n"))
+        assert len(csv) == 1
+        assert form.description.data in csv[0]
+
+
 def test_browse_filtering_filters_bad(client, mockdata, session):
     with current_app.test_request_context():
         race_list = ["BLACK", "WHITE"]
@@ -964,8 +1044,8 @@ def test_browse_filtering_allows_good(client, mockdata, session):
         form = BrowseForm(race='WHITE',
                           gender='M',
                           rank='COMMANDER',
-                          min_age=datetime.datetime.now().year - 1991,
-                          max_age=datetime.datetime.now().year - 1989)
+                          min_age=datetime.now().year - 1991,
+                          max_age=datetime.now().year - 1989)
 
         data = process_form_data(form.data)
 
@@ -984,7 +1064,6 @@ def test_browse_filtering_allows_good(client, mockdata, session):
         filter_list = rv.data.split("<dt>Gender</dt>")[1:]
         assert any("<dd>M</dd>" in token for token in filter_list)
 
-
 # def test_find_form_submission(client, mockdata):
 #     with current_app.test_request_context():
 #         form = FindOfficerForm()