Changes connection to AWS cluster, reformats queries with python elas…

…ticsearch

Pipfile

@@ -25,6 +25,9 @@ websockets = "==8.1"
 requests = "*"
 pytest = "*"
 pipx = "*"
+boto3 = "*"
+elasticsearch = "*"
+requests-aws4auth = "*"
 
 [requires]
 python_version = "3.7"

requirements.txt

@@ -1,3 +1,5 @@
+boto3
+elasticsearch
 click==7.1.2
 fastapi==0.55.1
 gunicorn==20.0.4
@@ -10,6 +12,7 @@ pytest
 python-dateutil==2.8.1
 pytz==2020.1
 requests
+requests-aws4auth
 six==1.15.0
 starlette==0.13.2
 tabulate==0.8.7

src/app/main.py

@@ -3,7 +3,7 @@
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import HTMLResponse
 from .queries import get_all_jobs, search_city_state, search_state, search_all_locations
-from .models import Search, Track
+from .models import Search, Track, User, All
 
 app = FastAPI()
 
@@ -38,7 +38,7 @@ async def search_all():
     return all
 
 
-@app.post("/search/")
+@app.post("/search")
 async def search_custom(search: Search):
     """
     Endpoint to return custom search when user specifies
@@ -49,16 +49,21 @@ async def search_custom(search: Search):
     """
 
     if (search.city == None) and (search.state == None):
-        return search_all_locations(search.search)
+        return search_all_locations(search.search, search.lim)
     elif search.city == None:
-        return search_state(search.search, search.state)
+        return search_state(search.search, search.state, search.lim)
     elif search.state == None:
         return {"error": "City must be accompanied by a state"}
     else:
-        return search_city_state(search.search, search.city, search.state)
+        return search_city_state(search.search, search.city, search.state, search.lim)
 
+@app.post("/user")
+async def search_user(user: User):
+    """
+
+    """
 
-@app.post("/track/")
+@app.post("/track")
 async def search_by_track(track: Track):
     """ 
     Simple endpoint to return jobs recommended for

src/app/models.py

@@ -1,13 +1,26 @@
 from pydantic import BaseModel
+from typing import List, Dict
 
+class All(BaseModel):
+    lim: int = 20
 
 class Search(BaseModel):
     search: str
     city: str = None
     state: str = None
+    lim: int = 20
 
 
 class Track(BaseModel):
     track: str
     city: str = None
     state: str = None
+    lim = 20
+
+class User(BaseModel):
+    user_id: int
+    track: str
+    skills: str
+    city: str
+    state: str
+    lim: int = 20

src/app/queries.py

@@ -1,48 +1,49 @@
 import requests
 import json
 import os
-
-es_name = os.environ["ES_NAME"]
-es_pass = os.environ["ES_PASS"]
-es_endpoint = os.environ["ES_ENDPOINT"]
-
-
-def connect(query):
-    """
-    Defines elasticsearch connection
-    Connects to job index and search API
-    
-    Param query - 
-    the query to be ran
-
-    Output -
-    Elasticsearch response in json format
-    """
-    uri = f"https://{es_name}:{es_pass}@{es_endpoint}/jobs/_search"
-    headers = {"Content-Type": "application/json"}
-
-    response = requests.get(uri, headers=headers, data=query)
-    return response.json()
-
+import boto3
+from requests_aws4auth import AWS4Auth
+from elasticsearch import Elasticsearch, RequestsHttpConnection
+
+host = os.environ["AWS_ENDPOINT"]
+region = os.environ["REGION"]
+
+service = "es"
+credentials = boto3.Session().get_credentials()
+awsauth = AWS4Auth(
+    credentials.access_key,
+    credentials.secret_key,
+    region,
+    service,
+    session_token=credentials.token,
+)
+
+es = Elasticsearch(
+    hosts=[host],
+    # http_auth=awsauth,
+    use_ssl=True,
+    verify_certs=True,
+    connection_class=RequestsHttpConnection,
+)
 
 def reformat(response_query):
     """
     Reformats elasticsearch query to remove extra information
     """
 
     data = list()
-    for hit in response_query["hits"]["hits"]:
+    for hit in response_query["hits"]["hits"][2:]:
         data.append(
             {
                 "id": hit["_id"],
-                "source_url": hit["_source"]["post_url"],
-                "title": hit["_source"]["title"],
-                "company": hit["_source"]["company"],
-                "description": hit["_source"]["description"],
-                "date_published": hit["_source"]["publication_date"],
-                "location_city": hit["_source"]["location_city"],
-                "location_state": hit["_source"]["location_state"],
-                "geo_locat": hit["_source"]["location_point"],
+                "source_url": hit["_source"]["doc"]["post_url"],
+                "title": hit["_source"]["doc"]["title"],
+                "company": hit["_source"]["doc"]["company"],
+                "description": hit["_source"]["doc"]["description"],
+                "date_published": hit["_source"]["doc"]["publication_date"],
+                "location_city": hit["_source"]["doc"]["location_city"],
+                "location_state": hit["_source"]["doc"]["location_state"],
+                "geo_locat": hit["_source"]["doc"]["location_point"],
             }
         )
 
@@ -54,13 +55,13 @@ def get_all_jobs():
 
     query = json.dumps({"query": {"match_all": {}}})
 
-    response = connect(query)
+    response = es.search(body=query, size=50)
     reformatted = reformat(response)
 
     return reformatted
 
 
-def search_all_locations(search):
+def search_all_locations(search, lim):
     """
     Query to use if user does not specify a location
     Does a multi_match for the search string in the 
@@ -75,13 +76,13 @@ def search_all_locations(search):
         }
     )
 
-    response = connect(query)
+    response = es.search(body=query, size=lim)
     reformatted = reformat(response)
 
     return reformatted
 
 
-def search_city_state(search, city, state):
+def search_city_state(search, city, state, lim):
     """
     Query to call if user specifies the location 
     they want to search in. 
@@ -96,12 +97,10 @@ def search_city_state(search, city, state):
             "query": {
                 "bool": {
                     "must": [
-                        {
-                            "multi_match": {
+                        {"multi_match": {
                                 "query": search,
                                 "fields": ["description, ", "title"],
-                            }
-                        }
+                                }}
                     ],
                     "should": [
                         {"match": {"location_city": city}},
@@ -112,13 +111,13 @@ def search_city_state(search, city, state):
         }
     )
 
-    response = connect(query)
+    response = es.search(body=query, size=lim)
     reformatted = reformat(response)
 
     return reformatted
 
 
-def search_state(search, state):
+def search_state(search, state, lim):
     """
     Query to use if user just specifies the state
     that they want to search in
@@ -142,8 +141,7 @@ def search_state(search, state):
         }
     )
 
-    print(query)
-    response = connect(query)
+    response = es.search(body=query, size=lim)
     reformatted = reformat(response)
 
     return reformatted