Fixed some data cleaning for incorrect -2 values based on Geetha's an…

…swers to our questions. Added species id column to citizen data. Merged mango varieties. Reran data cleaning to update citizen data csv files
BU-Spark · Jun 11, 2024 · 5451873 · 5451873
1 parent 8a69b50
commit 5451873
Show file tree

Hide file tree

Showing 38 changed files with 669,740 additions and 643,082 deletions.
diff --git a/-2_values.ipynb b/-2_values.ipynb
diff --git a/all data/citizen/andaman_and_nicobar_islands.csv b/all data/citizen/andaman_and_nicobar_islands.csv
diff --git a/all data/citizen/andhra_pradesh.csv b/all data/citizen/andhra_pradesh.csv
diff --git a/all data/citizen/arunachal_pradesh.csv b/all data/citizen/arunachal_pradesh.csv
diff --git a/all data/citizen/assam.csv b/all data/citizen/assam.csv
diff --git a/all data/citizen/bihar.csv b/all data/citizen/bihar.csv
diff --git a/all data/citizen/chandigarh.csv b/all data/citizen/chandigarh.csv
@@ -1,3 +1,3 @@
-Observation_ID,Date_of_observation,User_id,User_Tree_id,Species_name,Lat,Long,State_name,Leaves_fresh,Leaves_mature,Leaves_old,Flowers_bud,Flowers_open,Flowers_male,Flowers_Female,Fruits_unripe,Fruits_ripe,Fruits_open,Year,Week
-608079.0,2022-08-24,24061.0,120375.0,Indian Laburnum-Cassia fistula,30.72778,76.77138,Chandigarh,2.0,2.0,0.0,1.0,2.0,-2.0,-2.0,0.0,0.0,-2.0,2022,31
-608082.0,2022-08-24,24061.0,120377.0,Pride of India-Lagerstroemia speciosa,30.73051,76.71466,Chandigarh,2.0,2.0,1.0,1.0,2.0,-2.0,-2.0,0.0,0.0,0.0,2022,31
+Observation_ID,Date_of_observation,User_id,User_Tree_id,Species_id,Species_name,Lat,Long,State_name,Leaves_fresh,Leaves_mature,Leaves_old,Flowers_bud,Flowers_open,Flowers_male,Flowers_Female,Fruits_unripe,Fruits_ripe,Fruits_open,Year,Week
+608079.0,2022-08-24,24061.0,120375.0,1020,Indian Laburnum-Cassia fistula,30.72778,76.77138,Chandigarh,2.0,2.0,0.0,1.0,2.0,-2.0,-2.0,0.0,0.0,-2.0,2022,31
+608082.0,2022-08-24,24061.0,120377.0,1045,Pride of India-Lagerstroemia speciosa,30.73051,76.71466,Chandigarh,2.0,2.0,1.0,1.0,2.0,-2.0,-2.0,0.0,0.0,0.0,2022,31
diff --git a/all data/citizen/chhattisgarh.csv b/all data/citizen/chhattisgarh.csv
diff --git a/all data/citizen/dadra_and_nagar_haveli.csv b/all data/citizen/dadra_and_nagar_haveli.csv
@@ -1,4 +1,4 @@
-Observation_ID,Date_of_observation,User_id,User_Tree_id,Species_name,Lat,Long,State_name,Leaves_fresh,Leaves_mature,Leaves_old,Flowers_bud,Flowers_open,Flowers_male,Flowers_Female,Fruits_unripe,Fruits_ripe,Fruits_open,Year,Week
-401140.0,2020-02-18,18858.0,86035.0,Indian Laburnum-Cassia fistula,11.84967,75.41732,Dadra and Nagar Haveli,0.0,0.0,0.0,2.0,2.0,-2.0,-2.0,0.0,2.0,-2.0,2020,6
-291485.0,2019-03-15,15977.0,32117.0,Mango (all varieties)-Mangifera indica,8.34187,77.04894,Dadra and Nagar Haveli,1.0,2.0,1.0,2.0,2.0,-2.0,-2.0,2.0,0.0,-2.0,2019,9
-291521.0,2019-03-15,15977.0,32150.0,Purple Bauhinia-Bauhinia purpurea,8.34187,77.04894,Dadra and Nagar Haveli,1.0,2.0,1.0,0.0,0.0,-2.0,-2.0,0.0,0.0,0.0,2019,9
+Observation_ID,Date_of_observation,User_id,User_Tree_id,Species_id,Species_name,Lat,Long,State_name,Leaves_fresh,Leaves_mature,Leaves_old,Flowers_bud,Flowers_open,Flowers_male,Flowers_Female,Fruits_unripe,Fruits_ripe,Fruits_open,Year,Week
+401140.0,2020-02-18,18858.0,86035.0,1020,Indian Laburnum-Cassia fistula,11.84967,75.41732,Dadra and Nagar Haveli,0.0,0.0,0.0,2.0,2.0,-2.0,-2.0,0.0,2.0,-2.0,2020,6
+291485.0,2019-03-15,15977.0,32117.0,1090,Mango (all varieties)-Mangifera indica,8.34187,77.04894,Dadra and Nagar Haveli,1.0,2.0,1.0,2.0,2.0,-2.0,-2.0,2.0,0.0,-2.0,2019,9
+291521.0,2019-03-15,15977.0,32150.0,1012,Purple Bauhinia-Bauhinia purpurea,8.34187,77.04894,Dadra and Nagar Haveli,1.0,2.0,1.0,0.0,0.0,-2.0,-2.0,0.0,0.0,0.0,2019,9
diff --git a/all data/citizen/delhi.csv b/all data/citizen/delhi.csv
diff --git a/all data/citizen/goa.csv b/all data/citizen/goa.csv
diff --git a/all data/citizen/gujarat.csv b/all data/citizen/gujarat.csv
diff --git a/all data/citizen/haryana.csv b/all data/citizen/haryana.csv
diff --git a/all data/citizen/himachal_pradesh.csv b/all data/citizen/himachal_pradesh.csv
diff --git a/all data/citizen/jammu_and_kashmir.csv b/all data/citizen/jammu_and_kashmir.csv
@@ -1,9 +1,9 @@
-Observation_ID,Date_of_observation,User_id,User_Tree_id,Species_name,Lat,Long,State_name,Leaves_fresh,Leaves_mature,Leaves_old,Flowers_bud,Flowers_open,Flowers_male,Flowers_Female,Fruits_unripe,Fruits_ripe,Fruits_open,Year,Week
-699375.0,2023-10-31,23271.0,147397.0,Apple-Malus domestica,33.72254,75.14871,Jammu and Kashmir,1.0,1.0,2.0,0.0,0.0,-2.0,-2.0,0.0,1.0,-2.0,2023,40
-581159.0,2022-03-26,20285.0,112515.0,Apricot tree-Prunus armeniaca,34.24561,77.16045,Jammu and Kashmir,0.0,0.0,1.0,2.0,0.0,-2.0,-2.0,0.0,0.0,-2.0,2022,11
-699373.0,2023-10-31,23271.0,147395.0,Apricot tree-Prunus armeniaca,33.72228,75.14874,Jammu and Kashmir,0.0,1.0,1.0,0.0,0.0,-2.0,-2.0,0.0,1.0,-2.0,2023,40
-325984.0,2019-06-15,20393.0,58180.0,Babool-Acacia nilotica,32.65307,74.80336,Jammu and Kashmir,2.0,1.0,0.0,0.0,0.0,-2.0,-2.0,2.0,1.0,-2.0,2019,21
-326013.0,2019-06-15,20393.0,58205.0,Devil's Tree-Alstonia scholaris,32.65292,74.80711,Jammu and Kashmir,2.0,1.0,1.0,0.0,0.0,-2.0,-2.0,2.0,2.0,1.0,2019,21
-325996.0,2019-06-15,20393.0,58193.0,Mast Tree-Monoon longifolium,32.65307,74.80336,Jammu and Kashmir,2.0,1.0,0.0,-2.0,0.0,-2.0,-2.0,0.0,0.0,-2.0,2019,21
-699379.0,2023-10-31,23271.0,147399.0,Peach tree-Prunus persica,33.72258,75.14869,Jammu and Kashmir,0.0,1.0,2.0,0.0,0.0,-2.0,-2.0,0.0,0.0,-2.0,2023,40
-699377.0,2023-10-31,23271.0,147398.0,Plum tree-Prunus domestica,33.72225,75.14865,Jammu and Kashmir,0.0,1.0,1.0,0.0,0.0,-2.0,-2.0,0.0,0.0,-2.0,2023,40
+Observation_ID,Date_of_observation,User_id,User_Tree_id,Species_id,Species_name,Lat,Long,State_name,Leaves_fresh,Leaves_mature,Leaves_old,Flowers_bud,Flowers_open,Flowers_male,Flowers_Female,Fruits_unripe,Fruits_ripe,Fruits_open,Year,Week
+699375.0,2023-10-31,23271.0,147397.0,1232,Apple-Malus domestica,33.72254,75.14871,Jammu and Kashmir,1.0,1.0,2.0,0.0,0.0,-2.0,-2.0,0.0,1.0,-2.0,2023,40
+581159.0,2022-03-26,20285.0,112515.0,1168,Apricot tree-Prunus armeniaca,34.24561,77.16045,Jammu and Kashmir,0.0,0.0,1.0,2.0,0.0,-2.0,-2.0,0.0,0.0,-2.0,2022,11
+699373.0,2023-10-31,23271.0,147395.0,1168,Apricot tree-Prunus armeniaca,33.72228,75.14874,Jammu and Kashmir,0.0,1.0,1.0,0.0,0.0,-2.0,-2.0,0.0,1.0,-2.0,2023,40
+325984.0,2019-06-15,20393.0,58180.0,1001,Babool-Acacia nilotica,32.65307,74.80336,Jammu and Kashmir,2.0,1.0,0.0,0.0,0.0,-2.0,-2.0,2.0,1.0,-2.0,2019,21
+326013.0,2019-06-15,20393.0,58205.0,1008,Devil's Tree-Alstonia scholaris,32.65292,74.80711,Jammu and Kashmir,2.0,1.0,1.0,0.0,0.0,-2.0,-2.0,2.0,2.0,1.0,2019,21
+325996.0,2019-06-15,20393.0,58193.0,1065,Mast Tree-Monoon longifolium,32.65307,74.80336,Jammu and Kashmir,2.0,1.0,0.0,-2.0,0.0,-2.0,-2.0,0.0,0.0,-2.0,2019,21
+699379.0,2023-10-31,23271.0,147399.0,1173,Peach tree-Prunus persica,33.72258,75.14869,Jammu and Kashmir,0.0,1.0,2.0,0.0,0.0,-2.0,-2.0,0.0,0.0,-2.0,2023,40
+699377.0,2023-10-31,23271.0,147398.0,1170,Plum tree-Prunus domestica,33.72225,75.14865,Jammu and Kashmir,0.0,1.0,1.0,0.0,0.0,-2.0,-2.0,0.0,0.0,-2.0,2023,40