From 133152f66ba0e1ae7994cb422f8a1762a35b47b7 Mon Sep 17 00:00:00 2001
From: tomx <tomx4096@gmail.com>
Date: Sun, 19 Aug 2018 21:12:31 +0200
Subject: [PATCH 1/4] routes for generating CSV data dumps

---
 OpenOversight/app/main/views.py | 73 ++++++++++++++++++++++++++++++++-
 1 file changed, 72 insertions(+), 1 deletion(-)

diff --git a/OpenOversight/app/main/views.py b/OpenOversight/app/main/views.py
index bd774d0a7..ec002178a 100644
--- a/OpenOversight/app/main/views.py
+++ b/OpenOversight/app/main/views.py
@@ -9,7 +9,7 @@
 from werkzeug import secure_filename
 
 from flask import (abort, render_template, request, redirect, url_for,
-                   flash, current_app, jsonify)
+                   flash, current_app, jsonify, Response)
 from flask_login import current_user, login_required, login_user
 
 from . import main
@@ -584,6 +584,77 @@ def submit_data():
         return render_template('submit_image.html', form=form, preferred_dept_id=preferred_dept_id)
 
 
+def check_input(str_input):
+    if str_input is None or str_input == "Not Sure":
+        return ""
+    else:
+        return str(str_input).replace(",", " ")  # no commas allowed
+
+
+@main.route('/download/department/<int:department_id>', methods=['GET'])
+@limiter.limit('5/minute')
+def download_dept_csv(department_id):
+    department = Department.query.filter_by(id=department_id).first()
+    records = Officer.query.filter_by(department_id=department_id).all()
+    if not department or not records:
+        abort(404)
+    dept_name = records[0].department.name.replace(" ", "_")
+    first_row = "id, last, first, middle, suffix, gender, "\
+                "race, born, employment_date, assignments\n"
+
+    assign_dict = {}
+    assign_records = Assignment.query.all()
+    for r in assign_records:
+        if r.officer_id not in assign_dict:
+            assign_dict[r.officer_id] = []
+        assign_dict[r.officer_id].append("(#%s %s %s %s %s)" % (check_input(r.star_no), check_input(r.rank), check_input(r.unit), check_input(r.star_date), check_input(r.resign_date)))
+
+    record_list = ["%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n" %
+                   (str(record.id),
+                    check_input(record.last_name),
+                    check_input(record.first_name),
+                    check_input(record.middle_initial),
+                    check_input(record.suffix),
+                    check_input(record.gender),
+                    check_input(record.race),
+                    check_input(record.birth_year),
+                    check_input(record.employment_date),
+                    " ".join(assign_dict.get(record.id, [])),
+                    ) for record in records]
+
+    csv_name = dept_name + "_Officers.csv"
+    csv = first_row + "".join(record_list)
+    csv_headers = {"Content-disposition": "attachment; filename=" + csv_name}
+    return Response(csv, mimetype="text/csv", headers=csv_headers)
+
+
+@main.route('/download/department/<int:department_id>/incidents', methods=['GET'])
+@limiter.limit('5/minute')
+def download_incidents_csv(department_id):
+    department = Department.query.filter_by(id=department_id).first()
+    records = Incident.query.filter_by(department_id=department.id).all()
+    if not department or not records:
+        abort(404)
+    dept_name = records[0].department.name.replace(" ", "_")
+    first_row = "id,report_num,date,description,location,licences,links,officers\n"
+
+    record_list = ["%s,%s,%s,%s,%s,%s,%s,%s\n" %
+                   (str(record.id),
+                    check_input(record.report_number),
+                    check_input(record.date),
+                    check_input(record.description),
+                    check_input(record.address),
+                    " ".join(map(lambda x: str(x), record.license_plates)),
+                    " ".join(map(lambda x: str(x), record.links)),
+                    " ".join(map(lambda x: str(x), record.officers)),
+                    ) for record in records]
+
+    csv_name = dept_name + "_Incidents.csv"
+    csv = first_row + "".join(record_list)
+    csv_headers = {"Content-disposition": "attachment; filename=" + csv_name}
+    return Response(csv, mimetype="text/csv", headers=csv_headers)
+
+
 @main.route('/upload/department/<int:department_id>', methods=['POST'])
 @limiter.limit('250/minute')
 def upload(department_id):

From b8a6e427f3d85c955cbee3ddfeb60218a33b93e7 Mon Sep 17 00:00:00 2001
From: tomx <tomx4096@gmail.com>
Date: Sun, 19 Aug 2018 21:50:20 +0200
Subject: [PATCH 2/4] test officer and incident csv generation

---
 .../routes/test_officer_and_department.py     | 81 ++++++++++++++++++-
 1 file changed, 80 insertions(+), 1 deletion(-)

diff --git a/OpenOversight/tests/routes/test_officer_and_department.py b/OpenOversight/tests/routes/test_officer_and_department.py
index 806cd026f..f598230ec 100644
--- a/OpenOversight/tests/routes/test_officer_and_department.py
+++ b/OpenOversight/tests/routes/test_officer_and_department.py
@@ -1,6 +1,7 @@
 # Routing and view tests
 import pytest
 import random
+from datetime import datetime
 from flask import url_for, current_app
 from ..conftest import AC_DEPT
 from OpenOversight.app.utils import dept_choices
@@ -11,7 +12,8 @@
 from OpenOversight.app.main.forms import (AssignmentForm, DepartmentForm,
                                           AddOfficerForm, AddUnitForm,
                                           EditOfficerForm, LinkForm,
-                                          EditDepartmentForm,)
+                                          EditDepartmentForm, IncidentForm,
+                                          LocationForm, LicensePlateForm)
 
 from OpenOversight.app.models import Department, Unit, Officer
 
@@ -872,6 +874,83 @@ def test_admin_can_add_new_officer_with_suffix(mockdata, client, session):
         assert officer.suffix == 'Jr'
 
 
+def test_officer_csv(mockdata, client, session):
+    with current_app.test_request_context():
+        login_admin(client)
+        department = random.choice(dept_choices())
+        links = [
+            LinkForm(url='http://www.pleasework.com', link_type='link').data,
+        ]
+        form = AddOfficerForm(first_name='CKtVwe2gqhAIc',
+                              last_name='FVkcjigWUeUyA',
+                              middle_initial='T',
+                              suffix='Jr',
+                              race='WHITE',
+                              gender='M',
+                              star_no=90009,
+                              rank='PO',
+                              department=department.id,
+                              birth_year=1910,
+                              links=links)
+        # add the officer
+        rv = client.post(
+            url_for('main.add_officer'),
+            data=process_form_data(form.data),
+            follow_redirects=True
+        )
+        # dump officer csv
+        rv = client.get(
+            url_for('main.download_dept_csv', department_id=department.id),
+            follow_redirects=True
+        )
+        # get csv entry matching officer last name
+        csv = filter(lambda row: form.last_name.data in row, rv.data.split("\n"))
+        assert len(csv) == 1
+        assert form.first_name.data in csv[0]
+        assert form.last_name.data in csv[0]
+        assert form.rank.data in csv[0]
+
+
+def test_incidents_csv(mockdata, client, session):
+    with current_app.test_request_context():
+        login_admin(client)
+        department = random.choice(dept_choices())
+        incident_date = datetime(2000, 5, 25, 1, 45)
+        report_number = '42'
+
+        address_form = LocationForm(street_name='ABCDE', city='FGHI', state='IA')
+        link_form = LinkForm(url='http://example.com', link_type='video')
+        license_plates_form = LicensePlateForm(state='AZ')
+        form = IncidentForm(
+            date_field=str(incident_date.date()),
+            time_field=str(incident_date.time()),
+            report_number=report_number,
+            description='Something happened',
+            department=str(department.id),
+            department_id=department.id,
+            address=address_form.data,
+            links=[link_form.data],
+            license_plates=[license_plates_form.data],
+            officers=[]
+        )
+        # add the incident
+        rv = client.post(
+            url_for('main.incident_api') + 'new',
+            data=process_form_data(form.data),
+            follow_redirects=True
+        )
+        assert "created" in rv.data
+        # dump incident csv
+        rv = client.get(
+            url_for('main.download_incidents_csv', department_id=department.id),
+            follow_redirects=True
+        )
+        # print(rv.data)
+        # get the csv entry with matching report number
+        csv = filter(lambda row: report_number in row, rv.data.split("\n"))
+        assert len(csv) == 1
+        assert form.description.data in csv[0]
+
 # def test_find_form_submission(client, mockdata):
 #     with current_app.test_request_context():
 #         form = FindOfficerForm()

From d1ab8e3dc7f73b085c3f11eb12625490f8444c03 Mon Sep 17 00:00:00 2001
From: tomx <tomx4096@gmail.com>
Date: Sun, 19 Aug 2018 22:29:39 +0200
Subject: [PATCH 3/4] better string representation for Location model

---
 OpenOversight/app/models.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/OpenOversight/app/models.py b/OpenOversight/app/models.py
index f8b2d748d..bb21f1f8c 100644
--- a/OpenOversight/app/models.py
+++ b/OpenOversight/app/models.py
@@ -237,6 +237,20 @@ def validate_zip_code(self, key, zip_code):
     def validate_state(self, key, state):
         return state_validator(state)
 
+    def __repr__(self):
+        if self.street_name and self.cross_street2:
+            return 'Intersection of {} and {}, {} {}'.format(
+                self.street_name, self.cross_street2, self.city, self.state)
+        elif self.street_name and self.cross_street1:
+            return 'Intersection of {} and {}, {} {}'.format(
+                self.street_name, self.cross_street1, self.city, self.state)
+        elif self.street_name and self.cross_street1 and self.cross_street2:
+            return 'Intersection of {} between {} and {}, {} {}'.format(
+                self.street_name, self.cross_street1, self.cross_street2,
+                self.city, self.state)
+        else:
+            return '{} {}'.format(self.city, self.state)
+
 
 class LicensePlate(db.Model):
     __tablename__ = 'license_plates'

From f4f8dd37f7b8302b998f86e91fe233dcbadbbc9e Mon Sep 17 00:00:00 2001
From: tomx <tomx4096@gmail.com>
Date: Mon, 20 Aug 2018 11:23:55 +0200
Subject: [PATCH 4/4] link to CSV directory in footer

---
 OpenOversight/app/main/views.py            |  6 ++++++
 OpenOversight/app/templates/all_depts.html | 22 ++++++++++++++++++++++
 OpenOversight/app/templates/base.html      |  3 ++-
 3 files changed, 30 insertions(+), 1 deletion(-)
 create mode 100644 OpenOversight/app/templates/all_depts.html

diff --git a/OpenOversight/app/main/views.py b/OpenOversight/app/main/views.py
index ec002178a..d35cfb712 100644
--- a/OpenOversight/app/main/views.py
+++ b/OpenOversight/app/main/views.py
@@ -655,6 +655,12 @@ def download_incidents_csv(department_id):
     return Response(csv, mimetype="text/csv", headers=csv_headers)
 
 
+@main.route('/download/all', methods=['GET'])
+def all_data():
+    departments = Department.query.all()
+    return render_template('all_depts.html', departments=departments)
+
+
 @main.route('/upload/department/<int:department_id>', methods=['POST'])
 @limiter.limit('250/minute')
 def upload(department_id):
diff --git a/OpenOversight/app/templates/all_depts.html b/OpenOversight/app/templates/all_depts.html
new file mode 100644
index 000000000..8efdcb6a4
--- /dev/null
+++ b/OpenOversight/app/templates/all_depts.html
@@ -0,0 +1,22 @@
+{% extends "base.html" %}
+{% import "bootstrap/wtf.html" as wtf %}
+{% block content %}
+
+<div class="container theme-showcase" role="main">
+
+  <!--div class="text-center frontpage-leads"-->
+  {% for dept in departments %}
+    <p>
+       {{ dept.name }}
+       <a href={{ url_for('main.download_dept_csv',department_id=dept.id) }}>officers.csv</a> 
+
+       {% if dept.incidents %}
+           <a href={{ url_for('main.download_incidents_csv',department_id=dept.id) }}>incidents.csv</a>
+       {% endif %}
+
+    </p>
+  {% endfor %}
+  <!--/div-->
+</div>
+
+{% endblock %}
diff --git a/OpenOversight/app/templates/base.html b/OpenOversight/app/templates/base.html
index 3f9d6e393..d9e68b062 100644
--- a/OpenOversight/app/templates/base.html
+++ b/OpenOversight/app/templates/base.html
@@ -101,7 +101,8 @@ <h5>OpenOversight</h5>
           <p class="font-weight-300">
             A product of Lucy Parsons Labs in Chicago, IL. <br>
             <a href="https://lucyparsonslabs.com/" target="_blank" class="btn-unstyled">Lucy Parsons Labs</a><br>
-            <a href="https://lucyparsonslabs.com/securedrop/" target="_blank" class="btn-unstyled">SecureDrop</a>
+            <a href="https://lucyparsonslabs.com/securedrop/" target="_blank" class="btn-unstyled">SecureDrop</a><br>
+            <a href="{{ url_for('main.all_data') }}" target="_blank" class="btn-unstyled">Download department data</a>
           </p>
         </div>