Testing for load

Pipfile

@@ -22,6 +22,7 @@ fastapi = "*"
 uvicorn = "*"
 gunicorn = "*"
 psycopg2 = "*"
+starlette = "*"
 
 [requires]
 python_version = "3.7"

requirements.txt

@@ -10,6 +10,7 @@ fastapi
 psycopg2
 uvicorn
 gunicorn
+starlette
 
 
 # test requirements

src/app/extract.py

@@ -20,20 +20,20 @@
 # This is where we can define the titles that we want to search for
 main_titles = [
     "data engineer",
-    "data scientist",
-    "data analytics",
-    "python",
-    "machine learning",
-    "sql",
-    "pandas",
-    "front end",
-    "back end",
-    "full stack",
-    "react",
-    "angular",
-    "vue",
-    "software engineer",
-    "software developer",
+    # "data scientist",
+    # "data analytics",
+    # "python",
+    # "machine learning",
+    # "sql",
+    # "pandas",
+    # "front end",
+    # "back end",
+    # "full stack",
+    # "react",
+    # "angular",
+    # "vue",
+    # "software engineer",
+    # "software developer",
 ]
 
 

src/app/load.py

@@ -25,7 +25,7 @@
 es = Elasticsearch(
     send_get_body_as="POST",
     hosts=[host],
-    http_auth=awsauth,
+    #http_auth=awsauth,
     use_ssl=True,
     verify_certs=True,
     connection_class=RequestsHttpConnection,
@@ -54,5 +54,4 @@ def gendata(df):
 
 def query(df):
 
-    print(df)
-    # print(bulk(es, gendata(df)))
+    return(bulk(es, gendata(df)))

src/app/main.py

@@ -36,14 +36,14 @@ def start_upload():  # async
     Start the cron task to upload new jobs to the elasticsearch database
     """
     df_adzuna = adzuna()
-    df_jobsearcher = jobsearcher()
+    #df_jobsearcher = jobsearcher()
     #df_monster = monster_scraper()
 
     transformed_adzuna = transform_df(df_adzuna)
-    transformed_jobsearcher = transform_df(df_jobsearcher)
+    #transformed_jobsearcher = transform_df(df_jobsearcher)
     #transformed_monstser = transform_df(df_monster)
 
     query(transformed_adzuna)
-    query(transformed_jobsearcher)
+    #query(transformed_jobsearcher)
     #query(transformed_monstser)
     return "Cron job complete"

tests/test_load.py

@@ -0,0 +1,43 @@
+from src.app.load import query
+import pandas as pd
+
+def test_query():
+    exam = {"id" :["JS927927c0a7eb091796b82aea8f3a0770459567e4c662b9d727a428ccdeea092a"],
+        "post_url" : ["https://neuvoo.com/job.php?id=0aee8ad3da98&oapply=org_v2020-06&source=kimblegroup_bulk&utm_source=partner&utm_medium=kimblegroup_bulk&puid=gddg3aefgadb3aebfdd83aef3aec3dedbaacda9f4da7fda8aea33de83ee3fbdbgbddacdc9ed37ddf9ddbfdd7"],
+        "title" : ["Systems Engineer, EPMS"],
+        "title_keyword" : ["systems engineer, epms"],
+        "tags" : [[
+            "data center facilities",
+            "ms access",
+            "network management",
+            "dns",
+            "systems engineer",
+            "ip",
+            "hands on",
+            "architecture",
+            "troubleshooting",
+            "monitoring",
+            "communications",
+            "york",
+            "field",
+            "project",
+            "technology",
+            "support"
+          ]],
+        "company" : ["albireo energy"],
+        "description" : [[
+            "Albireo Energy is seeking a Systems Engineer in support of technology systems at multiple data center facilities in New York, New Jersey and Delaware.",
+            "Setup and administer network management and monitoring architecture",
+            "Maintain and troubleshoot field communications (Modbus RTU, Modbus TCP & TCP/IP) Ability to perform hands-on work with field equipment installations and troubleshooting",
+            "Knowledge of IP, DNS, WINS, DHCP, SNMP, SMTP, FTP, HTTP protocols",
+            "Familiarity with MS Project and MS Access preferred"
+            ]],
+        "publication_date" : ["2020-06-10"],
+        "inserted_date": ["2020-06-10"],
+        "city" : ["Newark"],
+        "state" : ["Delaware"],
+        "latitude" : ["39.714507"],
+        "longitude": ["-75.738715"]}
+    df = pd.DataFrame(data=exam)
+    q = query(df)
+    assert q[0] == 1

tests/test_main.py

@@ -0,0 +1,13 @@
+from starlette.testclient import TestClient
+from src.app.main import app
+
+client = TestClient(app)
+
+def test_home():
+    response = client.get("/")
+    assert response.status_code == 200
+
+
+def test_start():
+    response = client.get("/start")
+    assert response.status_code == 200