Merge pull request BloomTech-Labs#11 from Lambda-School-Labs/mapping_fix

fixes nested doc object when loading objects into bulk api
kellischeuble · Jun 9, 2020 · 50a5ce6 · 50a5ce6
2 parents e99a19a + e1cdd0e
commit 50a5ce6
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 13 deletions.
diff --git a/etl/load.py b/etl/load.py
@@ -38,20 +38,19 @@ def gendata(df):
             "_op_type": "index",
             "_index": "jobs",
             "_id": row["id"],
-            "doc": {
-                "post_url": row["post_url"],
-                "title": row["title"],
-                "title_keyword": row["title_keyword"],
-                "tags": row["tags"],
-                "company": row["company"],
-                "description": row["description"],
-                "publication_date": row["publication_date"],
-                "location_city": row["city"],
-                "location_state": row["state"],
-                "location_point": f"{row['latitude']},{row['longitude']}",
-            },
+            "post_url": row["post_url"],
+            "title": row["title"],
+            "title_keyword": row["title_keyword"],
+            "tags": row["tags"],
+            "company": row["company"],
+            "description": row["description"],
+            "publication_date": row["publication_date"],
+            "location_city": row["city"],
+            "location_state": row["state"],
+            "location_point": f"{row['latitude']},{row['longitude']}",
         }
 
 
 def query(df):
+
     print(bulk(es, gendata(df)))
diff --git a/etl/transform.py b/etl/transform.py
@@ -57,4 +57,7 @@ def transform_df(df):
     for col in keyword_cols:
         df[col] = df[col].apply(keyword)
 
+    for col in ["latitude", "longitude"]:
+        df[col] = df[col].fillna(0)
+
     return df
diff --git a/requirements.txt b/requirements.txt
@@ -1,8 +1,10 @@
 pandas
 arrow
 beautifulsoup4
-elasticsearch[async]
+boto3
+elasticsearch
 requests
+requests-aws4auth
 flatten-dict
 
 # test requirements