Scraper scheduler (#126)

* Added staging env

* Update scheduler

README.md

@@ -50,7 +50,7 @@ Check the `#uplift-backend` channel for the `service-account-key.json` which sho
 
 To run the app:
 ````bash
-python app.py
+flask run
 ````
 
 To run the test suite:

app.py

@@ -1,11 +1,10 @@
-import logging, schedule, time
+import logging, schedule
 from flask import Flask, render_template
+from flask_apscheduler import APScheduler
 from flask_graphql import GraphQLView
 from graphene import Schema
 from graphql.utils import schema_printer
 from src.database import db_session, init_db
-from src.models.capacity import Capacity
-from src.models.openhours import OpenHours
 from src.schema import Query
 from src.scrapers.capacities_scraper import fetch_capacities
 from src.scrapers.reg_hours_scraper import fetch_reg_building, fetch_reg_facility
@@ -15,9 +14,13 @@
 
 app = Flask(__name__)
 app.debug = True
-
 schema = Schema(query=Query)
 
+# Scheduler
+scheduler = APScheduler()
+scheduler.init_app(app)
+scheduler.start()
+
 
 @app.route("/")
 def index():
@@ -32,6 +35,8 @@ def shutdown_session(exception=None):
     db_session.remove()
 
 
+# Scrape every 15 minutes
+@scheduler.task("interval", id="scrape_sheets", seconds=900)
 def scrape_sheets():
     logging.info("Scraping from sheets...")
 
@@ -55,10 +60,4 @@ def scrape_sheets():
     schema_file.close()
 
 if __name__ == "__main__":
-    app.run(host="127.0.0.1", port=5000)
-
-# Schedule the scraping to run every 10 minutes
-schedule.every(10).minutes.do(scrape_sheets)
-while True:
-    schedule.run_pending()
-    time.sleep(60)
+    app.run(host="0.0.0.0", port=5000)

requirements.txt

@@ -12,11 +12,12 @@ certifi==2018.8.24
 cfgv==3.3.1
 chardet==3.0.4
 charset-normalizer==3.2.0
-click==6.7
+click==8.0
 distlib==0.3.7
 fastdiff==0.3.0
 filelock==3.12.2
-Flask==1.0.2
+Flask==2.2.5
+Flask-APScheduler==1.13.1
 Flask-GraphQL==2.0.0
 Flask-Migrate==2.4.0
 Flask-Script==2.0.5
@@ -33,12 +34,12 @@ gunicorn==19.9.0
 identify==2.5.24
 idna==2.6
 importlib-metadata==6.7.0
-itsdangerous==0.24
+itsdangerous==2.0
 jedi==0.18.2
-Jinja2==2.10.1
+Jinja2==3.0
 lxml==4.9.2
 Mako==1.0.9
-MarkupSafe==1.1.1
+MarkupSafe==2.1.1
 marshmallow==3.0.0rc4
 marshmallow-sqlalchemy==0.16.2
 nodeenv==1.8.0
@@ -74,5 +75,5 @@ virtualenv==20.24.5
 wasmer==1.1.0
 wasmer-compiler-cranelift==1.1.0
 wcwidth==0.2.6
-Werkzeug==0.15.5
+Werkzeug==2.2.2
 zipp==3.15.0