diff --git a/README.md b/README.md index b581bac10..1f6553ca5 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,10 @@ If you want to serve the static JSON files via a local web server: ## Developing +### Maintaining database schema + +When `make import` is run, a number of postgres tables are created for importing the downloaded data. The schema of these tables are explicitly defined in the `dbschema` directory and may have to be updated in the future to accommodate future data. Columns that hold string data may not be sized large enough for future data. For example, if a name column accepts names of at most 20 characters and in the future, we have data where the name is 21 characters long, the data import will fail. When this occurs, we will have to update the corresponding schema file in `dbschema` to support more characters. Simply make the change and re-run `make import` to verify that it succeeds. + ### Checking output data changes This repository is used to generate data files that are used by the website. After `make process` is run, a `build` directory is generated containing the data files. This directory is checked in to the repository and later checked out when generating the website. After making code changes, it is important to compare the generated `build` directory against the `build` directory generated before the code changes and verify that changes from the code changes are as expected. @@ -105,6 +109,8 @@ The expected changes are excluded before generating digests for data in the `bui An additional script has been created to generate a report that enables comparing the totals for candidates. The script is `bin/report-candidates.py` and it generates `build/candidates.csv` and `build/candidates.xlsx`. The reports include a list of all the candidates and totals calculated multiple ways that should add up to the same number. +To ensure that database schema changes are visible in pull requests, the complete postgres schema is also saved to a `schema.sql` file in the `build` directory. Because the `build` directory is automatically re-built for each branch in a PR and committed to the repository, any change to the schema caused by a code change will be shown a difference in the `schema.sql` file when reviewing the PR. + ### Adding a calculator Each metric about a candidate is calculated independently. A metric might be diff --git a/build/_data/candidates/oakland/2018-11-06/pamela-price.json b/build/_data/candidates/oakland/2018-11-06/pamela-price.json index a6bb840a4..288016079 100644 --- a/build/_data/candidates/oakland/2018-11-06/pamela-price.json +++ b/build/_data/candidates/oakland/2018-11-06/pamela-price.json @@ -23,7 +23,7 @@ "supporting_money": { "contributions_received": 109782.9, "total_contributions": 109782.9, - "total_expenditures": 113921.91, + "total_expenditures": 114167.41, "total_loans_received": 34500.0, "total_supporting_independent": 0.0, "support_list": [ @@ -51,7 +51,7 @@ "Candidate Filing/Ballot Fees": 1000.0, "Campaign Literature and Mailings": 9979.27, "Postage, Delivery and Messenger Services": 487.47, - "Professional Services (Legal, Accounting)": 58574.84, + "Professional Services (Legal, Accounting)": 58770.34, "Information Technology Costs (Internet, E-mail)": 2019.74 }, "supporting_by_type": { @@ -76,6 +76,6 @@ ] }, "total_contributions": 109782.9, - "total_expenditures": 113921.91, + "total_expenditures": 114167.41, "total_loans_received": 34500.0 } diff --git a/build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json b/build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json index 2ab42b65f..161cd3361 100644 --- a/build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json +++ b/build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json @@ -21,8 +21,8 @@ "is_winner": null, "filer_id": 1463520, "supporting_money": { - "contributions_received": 16807.0, - "total_contributions": 16807.0, + "contributions_received": 17407.0, + "total_contributions": 17407.0, "total_expenditures": 31588.37, "total_loans_received": 0.0, "total_supporting_independent": 0.0, @@ -30,13 +30,13 @@ ], "contributions_by_type": { - "Individual": 14838.0, + "Individual": 14238.0, "Unitemized": 1469.0, - "Other (includes Businesses)": 500.0 + "Other (includes Businesses)": 1700.0 }, "contributions_by_origin": { "Out of State": 1200.0, - "Within Oakland": 10138.0, + "Within Oakland": 10738.0, "Within California": 4000.0 }, "total_small_contributions": 1719.0, @@ -58,7 +58,7 @@ ] }, - "total_contributions": 16807.0, + "total_contributions": 17407.0, "total_expenditures": 31588.37, "total_loans_received": 0.0 } diff --git a/build/_data/committees/1463520.json b/build/_data/committees/1463520.json index f2e4ba995..a460e1c9b 100644 --- a/build/_data/committees/1463520.json +++ b/build/_data/committees/1463520.json @@ -1,6 +1,6 @@ { "iec": false, - "total_contributions": 15338.0, + "total_contributions": 15938.0, "contributions": [ { "title": "Oakland November 7th, 2023 Special Election", @@ -720,13 +720,26 @@ { "title": "Oakland November 7th, 2023 Special Election", "Filer_ID": "1463520", - "Tran_Emp": "Families in Action for Justice", - "Tran_Occ": "Chief Executive Officer", - "Entity_Cd": "Individual", + "Tran_Emp": null, + "Tran_Occ": null, + "Entity_Cd": "Other (includes Businesses)", "Tran_Amt1": 600.0, "Tran_Date": "2023-11-13", - "Tran_NamF": "Kimi", - "Tran_NamL": "Kean", + "Tran_NamF": null, + "Tran_NamL": "Families in Action for Justice", + "Tran_Zip4": "94621", + "election_name": "oakland-2023" + }, + { + "title": "Oakland November 7th, 2023 Special Election", + "Filer_ID": "1463520", + "Tran_Emp": null, + "Tran_Occ": null, + "Entity_Cd": "Other (includes Businesses)", + "Tran_Amt1": 600.0, + "Tran_Date": "2023-11-13", + "Tran_NamF": null, + "Tran_NamL": "Families in Action for Justice", "Tran_Zip4": "94621", "election_name": "oakland-2023" }, diff --git a/build/_data/elections/oakland/2018-06-05.json b/build/_data/elections/oakland/2018-06-05.json index 140316d58..b39b3380d 100644 --- a/build/_data/elections/oakland/2018-06-05.json +++ b/build/_data/elections/oakland/2018-06-05.json @@ -57,7 +57,7 @@ "total_contributions": 15000.0 }, { - "name": "Oakland Athletics Baseball Company", + "name": "Service Employees International Union Local 1021 Issues PAC", "type": "Measure", "election_name": "oakland-june-2018", "total_contributions": 10000.0 diff --git a/build/_data/elections/oakland/2023-11-07.json b/build/_data/elections/oakland/2023-11-07.json index 31324e7f6..4e77c51b3 100644 --- a/build/_data/elections/oakland/2023-11-07.json +++ b/build/_data/elections/oakland/2023-11-07.json @@ -1,22 +1,22 @@ { - "total_contributions": 35457.0, + "total_contributions": 36057.0, "total_contributions_by_source": { "Out of State": 1400.0, - "Within Oakland": 19934.0, + "Within Oakland": 20534.0, "Within California": 11326.0 }, "contributions_by_type": { "Committee": 6000.0, - "Individual": 26160.0, + "Individual": 25560.0, "Unitemized": 3097.0, - "Other (includes Businesses)": 500.0 + "Other (includes Businesses)": 1700.0 }, "most_expensive_races": [ { "slug": "school-board-district-5", "type": "office", "title": "School Board District 5", - "amount": 35457.0 + "amount": 36057.0 } ], "largest_small_proportion": [ @@ -73,13 +73,13 @@ ], "top_contributors_for_offices": [ { - "name": "Service Employees International Union Local 1021 Candidate PAC", + "name": "UA Local 342", "type": "Office", "election_name": "oakland-2023", "total_contributions": 1200.0 }, { - "name": "Oakland Education Association PAC", + "name": "Service Employees International Union Local 1021 Candidate PAC", "type": "Office", "election_name": "oakland-2023", "total_contributions": 1200.0 @@ -108,7 +108,7 @@ "total_spending": 1200.0 }, { - "name": "Peralta Federation of Teachers COPE", + "name": "Families in Action for Justice", "type": "Office", "election_name": "oakland-2023", "total_spending": 1200.0 @@ -128,7 +128,7 @@ "total_spending": 1200.0 }, { - "name": "Peralta Federation of Teachers COPE", + "name": "Families in Action for Justice", "type": "Office", "election_name": "oakland-2023", "total_spending": 1200.0 diff --git a/build/_data/elections/oakland/2024-03-05.json b/build/_data/elections/oakland/2024-03-05.json index a6261f2ef..db9ca8892 100644 --- a/build/_data/elections/oakland/2024-03-05.json +++ b/build/_data/elections/oakland/2024-03-05.json @@ -48,13 +48,13 @@ ], "top_contributors_for_offices": [ { - "name": "Monique C Houston", + "name": "Steven Von Stade", "type": "Office", "election_name": "oakland-march-2024", "total_contributions": 600.0 }, { - "name": "Steven Von Stade", + "name": "Monique C Houston", "type": "Office", "election_name": "oakland-march-2024", "total_contributions": 600.0 diff --git a/build/_data/elections/oakland/2024-11-05.json b/build/_data/elections/oakland/2024-11-05.json index 8bd55992d..8440b5b44 100644 --- a/build/_data/elections/oakland/2024-11-05.json +++ b/build/_data/elections/oakland/2024-11-05.json @@ -79,19 +79,19 @@ ], "top_contributors_for_offices": [ { - "name": "International Association of Firefighters Local 55", + "name": "International Federation of Professional and Technical Engineers-Local 21 TJ Anthony PAC Fund", "type": "Office", "election_name": "oakland-2024", "total_contributions": 2400.0 }, { - "name": "International Federation of Professional and Technical Engineers-Local 21 TJ Anthony PAC Fund", + "name": "International Association of Firefighters Local 55", "type": "Office", "election_name": "oakland-2024", "total_contributions": 2400.0 }, { - "name": "Todd Scanlin", + "name": "Weylin White", "type": "Office", "election_name": "oakland-2024", "total_contributions": 1800.0 diff --git a/build/_data/totals.json b/build/_data/totals.json index b90fe44a3..f6a1f4e10 100644 --- a/build/_data/totals.json +++ b/build/_data/totals.json @@ -1032,7 +1032,7 @@ "total_contributions": 15000.0 }, { - "name": "Oakland Athletics Baseball Company", + "name": "Service Employees International Union Local 1021 Issues PAC", "type": "Measure", "election_name": "oakland-june-2018", "total_contributions": 10000.0 @@ -1311,24 +1311,24 @@ ] }, "oakland-2023": { - "total_contributions": 35457.0, + "total_contributions": 36057.0, "total_contributions_by_source": { "Out of State": 1400.0, - "Within Oakland": 19934.0, + "Within Oakland": 20534.0, "Within California": 11326.0 }, "contributions_by_type": { "Committee": 6000.0, - "Individual": 26160.0, + "Individual": 25560.0, "Unitemized": 3097.0, - "Other (includes Businesses)": 500.0 + "Other (includes Businesses)": 1700.0 }, "most_expensive_races": [ { "slug": "school-board-district-5", "type": "office", "title": "School Board District 5", - "amount": 35457.0 + "amount": 36057.0 } ], "largest_small_proportion": [ @@ -1385,13 +1385,13 @@ ], "top_contributors_for_offices": [ { - "name": "Service Employees International Union Local 1021 Candidate PAC", + "name": "UA Local 342", "type": "Office", "election_name": "oakland-2023", "total_contributions": 1200.0 }, { - "name": "Oakland Education Association PAC", + "name": "Service Employees International Union Local 1021 Candidate PAC", "type": "Office", "election_name": "oakland-2023", "total_contributions": 1200.0 @@ -1420,7 +1420,7 @@ "total_spending": 1200.0 }, { - "name": "Peralta Federation of Teachers COPE", + "name": "Families in Action for Justice", "type": "Office", "election_name": "oakland-2023", "total_spending": 1200.0 @@ -1440,7 +1440,7 @@ "total_spending": 1200.0 }, { - "name": "Peralta Federation of Teachers COPE", + "name": "Families in Action for Justice", "type": "Office", "election_name": "oakland-2023", "total_spending": 1200.0 @@ -1500,13 +1500,13 @@ ], "top_contributors_for_offices": [ { - "name": "Monique C Houston", + "name": "Steven Von Stade", "type": "Office", "election_name": "oakland-march-2024", "total_contributions": 600.0 }, { - "name": "Steven Von Stade", + "name": "Monique C Houston", "type": "Office", "election_name": "oakland-march-2024", "total_contributions": 600.0 @@ -1646,19 +1646,19 @@ ], "top_contributors_for_offices": [ { - "name": "International Association of Firefighters Local 55", + "name": "International Federation of Professional and Technical Engineers-Local 21 TJ Anthony PAC Fund", "type": "Office", "election_name": "oakland-2024", "total_contributions": 2400.0 }, { - "name": "International Federation of Professional and Technical Engineers-Local 21 TJ Anthony PAC Fund", + "name": "International Association of Firefighters Local 55", "type": "Office", "election_name": "oakland-2024", "total_contributions": 2400.0 }, { - "name": "Todd Scanlin", + "name": "Weylin White", "type": "Office", "election_name": "oakland-2024", "total_contributions": 1800.0 diff --git a/build/candidates.csv b/build/candidates.csv index 7a84d4f63..1331e421f 100644 --- a/build/candidates.csv +++ b/build/candidates.csv @@ -133,7 +133,7 @@ merika-goolsby.json,oakland-2024-11-05,Merika Goolsby,0.0,0.0,0.0 zac-unger.json,oakland-2024-11-05,Zac Unger,125078.55,125078.55,125078.55 christian-miguel-martinez.json,oakland-2023-11-07,Christian Miguel Martinez,0.0,0.0,0.0 alexandra-ritzie-hernandez.json,oakland-2023-11-07,Alexandra Ritzie-Hernandez,18650.0,18950.0,18950.0 -jorge-c-lerma.json,oakland-2023-11-07,Jorge C. Lerma,16807.0,16807.0,16807.0 +jorge-c-lerma.json,oakland-2023-11-07,Jorge C. Lerma,17407.0,17407.0,17407.0 michael-houston.json,oakland-2024-03-05,Michael Houston,17745.0,17745.0,17745.0 bruce-quan.json,oakland-2016-11-08,Bruce Quan,75623.14,75623.14,75623.14 roseann-torres.json,oakland-2016-11-08,Roseann Torres,42403.01,42403.01,42403.01 diff --git a/build/digests.json b/build/digests.json index 7d5bd58ea..73a4dc455 100644 --- a/build/digests.json +++ b/build/digests.json @@ -1,14 +1,14 @@ { "_total_contributions": { - "_total_contributions_by_source_with_unitemized_from_elections": 40688848.58, - "_total_contributions_by_source_with_unitemized_from_tickets": 40688848.58, - "_total_contributions_by_source_with_unitemized_from_totals": 40688848.58, - "_total_contributions_by_type_with_loans_from_elections": 40688848.58, - "_total_contributions_by_type_with_loans_from_tickets": 40688848.58, - "_total_contributions_by_type_with_loans_from_totals": 40688848.58, - "_total_contributions_from_elections": 40686887.58, - "_total_contributions_from_tickets": 40686887.58, - "_total_contributions_from_totals": 40686887.58, + "_total_contributions_by_source_with_unitemized_from_elections": 40689448.58, + "_total_contributions_by_source_with_unitemized_from_tickets": 40689448.58, + "_total_contributions_by_source_with_unitemized_from_totals": 40689448.58, + "_total_contributions_by_type_with_loans_from_elections": 40689448.58, + "_total_contributions_by_type_with_loans_from_tickets": 40689448.58, + "_total_contributions_by_type_with_loans_from_totals": 40689448.58, + "_total_contributions_from_elections": 40687487.58, + "_total_contributions_from_tickets": 40687487.58, + "_total_contributions_from_totals": 40687487.58, "oakland-2022": { "_total_contributions_by_source_with_unitemized_from_elections": 6418730.02, "_total_contributions_by_source_with_unitemized_from_tickets": 6418730.02, @@ -21,15 +21,15 @@ "_total_contributions_from_totals": 6412430.02 }, "oakland-2023": { - "_total_contributions_by_source_with_unitemized_from_elections": 35757.0, - "_total_contributions_by_source_with_unitemized_from_tickets": 35757.0, - "_total_contributions_by_source_with_unitemized_from_totals": 35757.0, - "_total_contributions_by_type_with_loans_from_elections": 35757.0, - "_total_contributions_by_type_with_loans_from_tickets": 35757.0, - "_total_contributions_by_type_with_loans_from_totals": 35757.0, - "_total_contributions_from_elections": 35457.0, - "_total_contributions_from_tickets": 35457.0, - "_total_contributions_from_totals": 35457.0 + "_total_contributions_by_source_with_unitemized_from_elections": 36357.0, + "_total_contributions_by_source_with_unitemized_from_tickets": 36357.0, + "_total_contributions_by_source_with_unitemized_from_totals": 36357.0, + "_total_contributions_by_type_with_loans_from_elections": 36357.0, + "_total_contributions_by_type_with_loans_from_tickets": 36357.0, + "_total_contributions_by_type_with_loans_from_totals": 36357.0, + "_total_contributions_from_elections": 36057.0, + "_total_contributions_from_tickets": 36057.0, + "_total_contributions_from_totals": 36057.0 }, "oakland-2024": { "_total_contributions_by_source_with_unitemized_from_elections": 17745.0, @@ -84,8 +84,8 @@ } }, "_total_expenditures": { - "_expenditures_by_type_from_candidates": 12909628.25, - "_total_expenditures_from_candidates": 13023462.87, + "_expenditures_by_type_from_candidates": 12909823.75, + "_total_expenditures_from_candidates": 13023708.37, "_total_opposing_from_candidates": 0, "_total_supporting_from_candidates": 0, "oakland-2014": { @@ -101,8 +101,8 @@ "_total_supporting_from_candidates": 0 }, "oakland-2018": { - "_expenditures_by_type_from_candidates": 2614112.86, - "_total_expenditures_from_candidates": 2630426.31, + "_expenditures_by_type_from_candidates": 2614308.36, + "_total_expenditures_from_candidates": 2630671.81, "_total_opposing_from_candidates": 0, "_total_supporting_from_candidates": 0 }, @@ -2620,9 +2620,9 @@ "build/_data/candidates/oakland/2018-11-06/pamela-price.json:opposing_money": "f8dd08bb7301c605db884d314877484e", "build/_data/candidates/oakland/2018-11-06/pamela-price.json:party_affiliation": "37a6259cc0c1dae299a7866489dff0bd", "build/_data/candidates/oakland/2018-11-06/pamela-price.json:photo_url": "da971a6c8f66abd28d60f1694f2112d4", - "build/_data/candidates/oakland/2018-11-06/pamela-price.json:supporting_money": "6fcad855ba11f7bef791fa0308f4dbfe", + "build/_data/candidates/oakland/2018-11-06/pamela-price.json:supporting_money": "df868d93af8a0c8af0232ccfde91d725", "build/_data/candidates/oakland/2018-11-06/pamela-price.json:total_contributions": "79c52bc0db992fe2de30754f0b74df7c", - "build/_data/candidates/oakland/2018-11-06/pamela-price.json:total_expenditures": "698cae27b78826297ef30f771cd5f95d", + "build/_data/candidates/oakland/2018-11-06/pamela-price.json:total_expenditures": "fda05d64b0890d215356a3b90b671ef7", "build/_data/candidates/oakland/2018-11-06/pamela-price.json:total_loans_received": "2538d7da8eac9d165b42ddd9fa48944b", "build/_data/candidates/oakland/2018-11-06/pamela-price.json:twitter_url": "0cac887351066c31eb32cf0bd9b4ac74", "build/_data/candidates/oakland/2018-11-06/pamela-price.json:votersedge_url": "2f9d9ee47e6891e0c548f9c2f20e77a0", @@ -4830,8 +4830,8 @@ "build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json:opposing_money": "afbcf2c18f150bac0905ceffb9c94bbc", "build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json:party_affiliation": "7936573151f1f9db4b3204b6eb1c3c6f", "build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json:photo_url": "3aebb0b5a2e5ece371e6ff29274a1606", - "build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json:supporting_money": "63a41e49647af8e0548ab7d9dcfaa284", - "build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json:total_contributions": "40b7a7f701d15c944bea7e5725f2ff51", + "build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json:supporting_money": "97378cb225b6f7412cf75576f18954d2", + "build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json:total_contributions": "d34e378b790bacc8825a32b75222dc8f", "build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json:total_expenditures": "5b71ea7186a5abc16768ef72488e8fbb", "build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json:total_loans_received": "30565a8911a6bb487e3745c0ea3c8224", "build/_data/candidates/oakland/2023-11-07/jorge-c-lerma.json:twitter_url": "1282de084fa43d1fafc646b7dd8794f5", @@ -6232,9 +6232,9 @@ "build/_data/committees/1463441.json:contributions": "141bc9ee1fafc430bbefe7cb18812fc1", "build/_data/committees/1463441.json:iec": "68934a3e9455fa72420237eb05902327", "build/_data/committees/1463441.json:total_contributions": "6f8f8c09f1d61e8f6c583841bf13d5ac", - "build/_data/committees/1463520.json:contributions": "87b99e394fc8074274f5bf0b3738d517", + "build/_data/committees/1463520.json:contributions": "b8f67708ae9856531b9cacf785a35fab", "build/_data/committees/1463520.json:iec": "68934a3e9455fa72420237eb05902327", - "build/_data/committees/1463520.json:total_contributions": "c27fec3252bb254defec2105142ddabc", + "build/_data/committees/1463520.json:total_contributions": "8197d4e762c43b3199c5acbb8b1a411a", "build/_data/committees/1463590.json:contributions": "355142d60745363d5608826786bca771", "build/_data/committees/1463590.json:iec": "68934a3e9455fa72420237eb05902327", "build/_data/committees/1463590.json:total_contributions": "4936e819885ed80f2d1186343930193f", @@ -6418,18 +6418,18 @@ "build/_data/elections/oakland/2022-11-08.json:top_spenders_for_offices": "6b06ae8f5ca85839bcea704a2cc2a496", "build/_data/elections/oakland/2022-11-08.json:total_contributions": "7ac6ca2fb8562e4458d2eb88a2551a2a", "build/_data/elections/oakland/2022-11-08.json:total_contributions_by_source": "96647cdc77d36bd25ba10e30161223c4", - "build/_data/elections/oakland/2023-11-07.json:contributions_by_type": "3d19755ec8d84ff944d11564df0240f5", + "build/_data/elections/oakland/2023-11-07.json:contributions_by_type": "8c33ddc28275311d996c758606a482af", "build/_data/elections/oakland/2023-11-07.json:largest_independent_expenditures": "c3ec617716307f60702698121b3b9b59", "build/_data/elections/oakland/2023-11-07.json:largest_small_proportion": "831f8d66b7a77eaf1018e9dd73518a32", - "build/_data/elections/oakland/2023-11-07.json:most_expensive_races": "ff4fc5089a6d80f15905fee6992177e3", + "build/_data/elections/oakland/2023-11-07.json:most_expensive_races": "67584411d3b631d7343895db4508a757", "build/_data/elections/oakland/2023-11-07.json:top_contributors": "033b6cbaca24715c7153e497af7a788b", "build/_data/elections/oakland/2023-11-07.json:top_contributors_for_measures": "d751713988987e9331980363e24189ce", "build/_data/elections/oakland/2023-11-07.json:top_contributors_for_offices": "5393947fbf113e7f50eb393663d246c5", "build/_data/elections/oakland/2023-11-07.json:top_spenders": "085ec8c96f3354ea505f66e52f839c83", "build/_data/elections/oakland/2023-11-07.json:top_spenders_for_measures": "d751713988987e9331980363e24189ce", "build/_data/elections/oakland/2023-11-07.json:top_spenders_for_offices": "085ec8c96f3354ea505f66e52f839c83", - "build/_data/elections/oakland/2023-11-07.json:total_contributions": "86581c11a3d9a08162387fe03a0a02c7", - "build/_data/elections/oakland/2023-11-07.json:total_contributions_by_source": "a84b38f44bbce666a23a8322120d03d3", + "build/_data/elections/oakland/2023-11-07.json:total_contributions": "8e68d2e7e3f0699ca140c50d11ae3de1", + "build/_data/elections/oakland/2023-11-07.json:total_contributions_by_source": "21ee0886c42dee139dd0120e0bcf923f", "build/_data/elections/oakland/2024-03-05.json:contributions_by_type": "14a0a2a0eab8b22202a9fa1b3b967c8b", "build/_data/elections/oakland/2024-03-05.json:largest_independent_expenditures": "99914b932bd37a50b983c5e7c90ae93b", "build/_data/elections/oakland/2024-03-05.json:largest_small_proportion": "d3664254c0bc0ff340acdef7f7c29577", @@ -7223,7 +7223,7 @@ "build/_data/totals.json:oakland-2018": "256a6669774b5ed6881fce204ffcb0ee", "build/_data/totals.json:oakland-2020": "6798c19107f4d5e8bde3fc48e9fb97b5", "build/_data/totals.json:oakland-2022": "0e89f2caa32318b847fb60c9830606b8", - "build/_data/totals.json:oakland-2023": "f795ab90faa748dea9c40008cf9a2e07", + "build/_data/totals.json:oakland-2023": "745b7a92719daf75e96c18171d5c22cc", "build/_data/totals.json:oakland-2024": "e70ffaf5557810fb1f2ea3b804f95ab2", "build/_data/totals.json:oakland-june-2018": "cad345c2d29c542c3556a7bafcc12337", "build/_data/totals.json:oakland-june-2022": "7981581bb67e6583b89af31a367d8378", diff --git a/download/.github/workflows/main.yml b/download/.github/workflows/main.yml new file mode 100644 index 000000000..4321ffe01 --- /dev/null +++ b/download/.github/workflows/main.yml @@ -0,0 +1,21 @@ +name: "Check Google Drive Access" +on: + workflow_dispatch: +jobs: + check: + runs-on: ubuntu-latest + env: + REPO_OWNER: ${{ github.repository_owner}} + REPO_BRANCH: ${{ github.ref_name }} + SERVICE_ACCOUNT_KEY_JSON: ${{ secrets.SERVICE_ACCOUNT_KEY_JSON }} + GDRIVE_FOLDER: ${{ vars.GDRIVE_FOLDER }} + steps: + - uses: actions/checkout@v3 + - run: "pip install -r gdrive_requirements.txt" + - run: "python test_pull_from_gdrive.py" + - name: Archive pulled files + uses: actions/upload-artifact@v2 + with: + name: redacted-netfile-files + path: .local/downloads + diff --git a/download/main.py b/download/main.py index ea71320ed..be808dfc6 100644 --- a/download/main.py +++ b/download/main.py @@ -3,8 +3,7 @@ from datetime import datetime import json import pandas as pd -import os -#from sqlalchemy import create_engine +from sqlalchemy import create_engine from model.a_contributions import A_Contributions from model.committee import Committees from model.election import Elections @@ -83,11 +82,9 @@ def main(): 'XRef_Match', ]).sample(n=20)) - to_csv_dirpath = 'downloads/csv' - os.makedirs(to_csv_dirpath, exist_ok=True) - elections.df.to_csv(f'{to_csv_dirpath}/elections_v2.csv', index=False) - committees.df.to_csv(f'{to_csv_dirpath}/committees_v2.csv', index=False) - a_contributions.df.to_csv(f'{to_csv_dirpath}/a_contributions_v2.csv', index=False) + elections.df.to_csv('.local/elections.csv', index=False) + committees.df.to_csv('.local/committees.csv', index=False) + a_contributions.df.to_csv('.local/a_contributions.csv', index=False) ''' with engine.connect() as conn: diff --git a/download/model/a_contributions.py b/download/model/a_contributions.py index 6283296ef..71c79b8ea 100644 --- a/download/model/a_contributions.py +++ b/download/model/a_contributions.py @@ -3,7 +3,7 @@ Hopefully this can be joined with other Schedule classes into a single Transaction class """ import pandas as pd -#from sqlalchemy.types import BOOLEAN, DATE, DOUBLE_PRECISION, INTEGER, TIME, VARCHAR +from sqlalchemy.types import BOOLEAN, DATE, DOUBLE_PRECISION, INTEGER, TIME, VARCHAR from .base import BaseModel class A_Contributions(BaseModel): @@ -123,82 +123,82 @@ def __init__( 'Loan_Rate': 'string', 'Int_CmteId': 'Int64' } - #self._sql_dtypes = { - # 'Filer_ID': VARCHAR(9), - # 'Filer_NamL': VARCHAR(183), - # 'Report_Num': INTEGER, - # 'Committee_Type': VARCHAR(64), - # 'Rpt_Date': DATE, - # 'From_Date': DATE, - # 'Thru_Date': DATE, - # 'Elect_Date': DATE, - # 'tblCover_Office_Cd': VARCHAR(64), - # 'tblCover_Offic_Dscr': VARCHAR(64), - # 'Rec_Type': VARCHAR(4), - # 'Form_Type': TIME, - # 'Tran_ID': VARCHAR(12), - # 'Entity_Cd': VARCHAR(3), - # 'Tran_NamL': VARCHAR(199), - # 'Tran_NamF': VARCHAR(38), - # 'Tran_NamT': VARCHAR(6), - # 'Tran_NamS': VARCHAR(5), - # 'Tran_Adr1': VARCHAR(64), - # 'Tran_Adr2': VARCHAR(64), - # 'Tran_City': VARCHAR(50), - # 'Tran_State': VARCHAR(4), - # 'Tran_Zip4': VARCHAR(10), - # 'Tran_Emp': VARCHAR(92), - # 'Tran_Occ': VARCHAR(60), - # 'Tran_Self': BOOLEAN, - # 'Tran_Type': VARCHAR(4), - # 'Tran_Date': DATE, - # 'Tran_Date1': DATE, - # 'Tran_Amt1': DOUBLE_PRECISION, - # 'Tran_Amt2': DOUBLE_PRECISION, - # 'Tran_Dscr': VARCHAR(56), - # 'Cmte_ID': VARCHAR(9), - # 'Tres_NamL': VARCHAR(4), - # 'Tres_NamF': VARCHAR(4), - # 'Tres_NamT': VARCHAR(64), - # 'Tres_NamS': VARCHAR(64), - # 'Tres_Adr1': VARCHAR(64), - # 'Tres_Adr2': VARCHAR(64), - # 'Tres_City': VARCHAR(7), - # 'Tres_State': VARCHAR(4), - # 'Tres_Zip': INTEGER, - # 'Intr_NamL': VARCHAR(74), - # 'Intr_NamF': VARCHAR(6), - # 'Intr_NamT': VARCHAR(64), - # 'Intr_NamS': VARCHAR(64), - # 'Intr_Adr1': VARCHAR(64), - # 'Intr_Adr2': VARCHAR(64), - # 'Intr_City': VARCHAR(13), - # 'Intr_State': VARCHAR(4), - # 'Intr_Zip4': VARCHAR(10), - # 'Intr_Emp': VARCHAR(15), - # 'Intr_Occ': VARCHAR(8), - # 'Intr_Self': BOOLEAN, - # 'Cand_NamL': VARCHAR(64), - # 'Cand_NamF': VARCHAR(64), - # 'Cand_NamT': VARCHAR(64), - # 'Cand_NamS': VARCHAR(64), - # 'tblDetlTran_Office_Cd': VARCHAR(4), - # 'tblDetlTran_Offic_Dscr': VARCHAR(19), - # 'Juris_Cd': VARCHAR(4), - # 'Juris_Dscr': VARCHAR(64), - # 'Dist_No': VARCHAR(64), - # 'Off_S_H_Cd': VARCHAR(64), - # 'Bal_Name': VARCHAR(64), - # 'Bal_Num': VARCHAR(4), - # 'Bal_Juris': VARCHAR(64), - # 'Sup_Opp_Cd': VARCHAR(64), - # 'Memo_Code': VARCHAR(64), - # 'Memo_RefNo': VARCHAR(11), - # 'BakRef_TID': VARCHAR(64), - # 'XRef_SchNm': VARCHAR(64), - # 'XRef_Match': VARCHAR(64), - # 'Loan_Rate': VARCHAR(64), - # 'Int_CmteId': INTEGER - #} - #self._sql_cols = list(self._sql_dtypes.keys()) + self._sql_dtypes = { + 'Filer_ID': VARCHAR(9), + 'Filer_NamL': VARCHAR(183), + 'Report_Num': INTEGER, + 'Committee_Type': VARCHAR(64), + 'Rpt_Date': DATE, + 'From_Date': DATE, + 'Thru_Date': DATE, + 'Elect_Date': DATE, + 'tblCover_Office_Cd': VARCHAR(64), + 'tblCover_Offic_Dscr': VARCHAR(64), + 'Rec_Type': VARCHAR(4), + 'Form_Type': TIME, + 'Tran_ID': VARCHAR(12), + 'Entity_Cd': VARCHAR(3), + 'Tran_NamL': VARCHAR(199), + 'Tran_NamF': VARCHAR(38), + 'Tran_NamT': VARCHAR(6), + 'Tran_NamS': VARCHAR(5), + 'Tran_Adr1': VARCHAR(64), + 'Tran_Adr2': VARCHAR(64), + 'Tran_City': VARCHAR(50), + 'Tran_State': VARCHAR(4), + 'Tran_Zip4': VARCHAR(10), + 'Tran_Emp': VARCHAR(92), + 'Tran_Occ': VARCHAR(60), + 'Tran_Self': BOOLEAN, + 'Tran_Type': VARCHAR(4), + 'Tran_Date': DATE, + 'Tran_Date1': DATE, + 'Tran_Amt1': DOUBLE_PRECISION, + 'Tran_Amt2': DOUBLE_PRECISION, + 'Tran_Dscr': VARCHAR(56), + 'Cmte_ID': VARCHAR(9), + 'Tres_NamL': VARCHAR(4), + 'Tres_NamF': VARCHAR(4), + 'Tres_NamT': VARCHAR(64), + 'Tres_NamS': VARCHAR(64), + 'Tres_Adr1': VARCHAR(64), + 'Tres_Adr2': VARCHAR(64), + 'Tres_City': VARCHAR(7), + 'Tres_State': VARCHAR(4), + 'Tres_Zip': INTEGER, + 'Intr_NamL': VARCHAR(74), + 'Intr_NamF': VARCHAR(6), + 'Intr_NamT': VARCHAR(64), + 'Intr_NamS': VARCHAR(64), + 'Intr_Adr1': VARCHAR(64), + 'Intr_Adr2': VARCHAR(64), + 'Intr_City': VARCHAR(13), + 'Intr_State': VARCHAR(4), + 'Intr_Zip4': VARCHAR(10), + 'Intr_Emp': VARCHAR(15), + 'Intr_Occ': VARCHAR(8), + 'Intr_Self': BOOLEAN, + 'Cand_NamL': VARCHAR(64), + 'Cand_NamF': VARCHAR(64), + 'Cand_NamT': VARCHAR(64), + 'Cand_NamS': VARCHAR(64), + 'tblDetlTran_Office_Cd': VARCHAR(4), + 'tblDetlTran_Offic_Dscr': VARCHAR(19), + 'Juris_Cd': VARCHAR(4), + 'Juris_Dscr': VARCHAR(64), + 'Dist_No': VARCHAR(64), + 'Off_S_H_Cd': VARCHAR(64), + 'Bal_Name': VARCHAR(64), + 'Bal_Num': VARCHAR(4), + 'Bal_Juris': VARCHAR(64), + 'Sup_Opp_Cd': VARCHAR(64), + 'Memo_Code': VARCHAR(64), + 'Memo_RefNo': VARCHAR(11), + 'BakRef_TID': VARCHAR(64), + 'XRef_SchNm': VARCHAR(64), + 'XRef_Match': VARCHAR(64), + 'Loan_Rate': VARCHAR(64), + 'Int_CmteId': INTEGER + } + self._sql_cols = list(self._sql_dtypes.keys()) self._sql_table_name = 'A-Contributions' diff --git a/download/model/base.py b/download/model/base.py index 999b20e94..9f8d50b79 100644 --- a/download/model/base.py +++ b/download/model/base.py @@ -20,10 +20,7 @@ def data(self): def df(self): """ Get a dataframe of the data """ if self._df is None or self._df.empty: - if len(self._data) > 0: - self._df = pd.DataFrame(self._data).astype(self._dtypes) - else: - self._df = pd.DataFrame(columns=self._dtypes.keys()) + self._df = pd.DataFrame(self._data).astype(self._dtypes) return self._df diff --git a/download/model/transaction.py b/download/model/transaction.py index e2c572364..a1a121cc2 100644 --- a/download/model/transaction.py +++ b/download/model/transaction.py @@ -1,8 +1,6 @@ """ Transactions """ from .base import BaseModel -import logging - class Transactions(BaseModel): """ A collection of transactions """ def __init__(self, transactions): @@ -75,13 +73,9 @@ def __init__(self, transactions): 'Tres_Zip': tran['tresZip4'], 'XRef_SchNm': tran['xrefSchNum'], 'XRef_Match': tran['xrefMatch'] - } for t in transactions if t['transaction'] + } for t in transactions ]) - missing_transactions = [t for t in transactions if t['transaction'] is None] - if len(missing_transactions) > 0: - logging.warn(f'{len(missing_transactions)} records missing transactions') - self._dtypes = { 'filing_nid': 'string', 'Rec_Type': 'string', diff --git a/download/requirements.txt b/download/requirements.txt index 2c639eaa7..c90e61684 100644 --- a/download/requirements.txt +++ b/download/requirements.txt @@ -1,4 +1,4 @@ -# The following version causes csvsql to fail: -#SQLAlchemy~=2.0.20 -# Do we need to upgrade the following? -#psycopg2~=2.9.7 +pandas~=2.0.3 +SQLAlchemy~=2.0.20 +psycopg2~=2.9.7 +gdrive-datastore==0.0.1.5 diff --git a/download/test_pull_from_gdrive.py b/download/test_pull_from_gdrive.py new file mode 100644 index 000000000..6b4d83df4 --- /dev/null +++ b/download/test_pull_from_gdrive.py @@ -0,0 +1,4 @@ +import os +from gdrive_datastore.gdrive import pull_data + +pull_data(subfolder='main', default_folder='OpenDisclosure') diff --git a/requirements.txt b/requirements.txt index 46573204e..7e458a0c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ awesome-slugify==1.6.5 awscli>=1.16.89 -Babel==2.13.1 +Babel==2.9.1 csvkit==1.4.0 dbfread==2.0.7 et-xmlfile==1.0.1