Merge pull request #171 from cuappdev/migrate-dev-server

Migration: moved scrapers under condition

.github/workflows/deploy-dev.yml

@@ -39,11 +39,10 @@ jobs:
             touch tags
             source tags
             export IMAGE_TAG=${{ steps.vars.outputs.sha_short }}
-            echo "export IMAGE_TAG=${IMAGE_TAG}" > tags
             cd docker-compose
             docker stack rm the-stack
             sleep 20s
             sudo systemctl stop nginx
             sudo systemctl restart nginx
             docker stack deploy -c docker-compose.yml the-stack --with-registry-auth
-            yes | docker system prune -a
+            yes | docker system prune -a

app.py

@@ -9,13 +9,6 @@
 from src.database import Base as db
 from flask_migrate import Migrate
 from src.schema import Query, Mutation
-from src.scrapers.capacities_scraper import fetch_capacities
-from src.scrapers.reg_hours_scraper import fetch_reg_building, fetch_reg_facility
-from src.scrapers.scraper_helpers import clean_past_hours
-from src.scrapers.sp_hours_scraper import fetch_sp_facility
-from src.scrapers.equipment_scraper import scrape_equipment
-from src.scrapers.class_scraper import fetch_classes
-from src.scrapers.activities_scraper import fetch_activity
 from src.utils.utils import create_gym_table
 from src.models.openhours import OpenHours
 from src.database import db_url, db_user, db_password, db_name, db_host, db_port
@@ -45,8 +38,6 @@
 app.config['SQLALCHEMY_DATABASE_URI'] = db_url
 app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
 
-migrate = Migrate(app, db)
-
 schema = Schema(query=Query, mutation=Mutation)
 swagger = Swagger(app)
 
@@ -72,52 +63,58 @@ def shutdown_session(exception=None):
     db_session.remove()
 
 
-# Scrape hours every 15 minutes
-@scheduler.task("interval", id="scrape_hours", seconds=900)
-def scrape_hours():
-    logging.info("Scraping hours from sheets...")
-
-    # Clear hours
-    db_session.query(OpenHours).delete()
+# Create schema.graphql
+with open("schema.graphql", "w+") as schema_file:
+    schema_file.write(schema_printer.print_schema(schema))
+    schema_file.close()
 
-    fetch_reg_facility()
-    fetch_reg_building()
-    fetch_sp_facility()
-    clean_past_hours()
+if __name__ == "__main__":
+    from src.scrapers.capacities_scraper import fetch_capacities
+    from src.scrapers.reg_hours_scraper import fetch_reg_building, fetch_reg_facility
+    from src.scrapers.scraper_helpers import clean_past_hours
+    from src.scrapers.sp_hours_scraper import fetch_sp_facility
+    from src.scrapers.equipment_scraper import scrape_equipment
+    from src.scrapers.class_scraper import fetch_classes
+    from src.scrapers.activities_scraper import fetch_activity
+    # Scrape hours every 15 minutes
 
+    @scheduler.task("interval", id="scrape_hours", seconds=900)
+    def scrape_hours():
+        logging.info("Scraping hours from sheets...")
 
-# Scrape capacities every 10 minutes
-@scheduler.task("interval", id="scrape_capacities", seconds=600)
-def scrape_capacities():
-    logging.info("Scraping capacities from C2C...")
+        # Clear hours
+        db_session.query(OpenHours).delete()
 
-    fetch_capacities()
+        fetch_reg_facility()
+        fetch_reg_building()
+        fetch_sp_facility()
+        clean_past_hours()
 
+    # Scrape capacities every 10 minutes
 
-# Scrape classes every hour
-@scheduler.task("interval", id="scrape_classes", seconds=3600)
-def scrape_classes():
-    logging.info("Scraping classes from group-fitness-classes...")
+    @scheduler.task("interval", id="scrape_capacities", seconds=600)
+    def scrape_capacities():
+        logging.info("Scraping capacities from C2C...")
 
-    fetch_classes(10)
+        fetch_capacities()
 
+    # Scrape classes every hour
 
-# Create database and fill it with data
-init_db()
-create_gym_table()
+    @scheduler.task("interval", id="scrape_classes", seconds=3600)
+    def scrape_classes():
+        logging.info("Scraping classes from group-fitness-classes...")
 
-scrape_classes()
-scrape_hours()
-scrape_capacities()
-scrape_equipment()
+        fetch_classes(10)
 
-logging.info("Scraping activities from sheets...")
-fetch_activity()
+    # Create database and fill it with data
+    init_db()
+    create_gym_table()
 
-# Create schema.graphql
-with open("schema.graphql", "w+") as schema_file:
-    schema_file.write(schema_printer.print_schema(schema))
-    schema_file.close()
+    scrape_classes()
+    scrape_hours()
+    scrape_capacities()
+    scrape_equipment()
+    logging.info("Scraping activities from sheets...")
+    fetch_activity()
 
-if __name__ == "__main__":
     app.run(host="127.0.0.1", port=5000)

migrations/env.py

@@ -1,4 +1,5 @@
 from __future__ import with_statement
+from flask import current_app
 
 import logging
 from logging.config import fileConfig
@@ -21,7 +22,6 @@
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-from flask import current_app
 config.set_main_option('sqlalchemy.url',
                        current_app.config.get('SQLALCHEMY_DATABASE_URI'))
 target_metadata = current_app.extensions['migrate'].db.metadata

migrations/versions/f711f3c11324_initial_migration.py

@@ -19,14 +19,18 @@
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
     op.alter_column('gear', 'cost',
-               existing_type=postgresql.DOUBLE_PRECISION(precision=53),
-               nullable=0)
+                    existing_type=postgresql.DOUBLE_PRECISION(precision=53),
+                    nullable=0)
+    op.alter_column('equipment', 'categories',
+                    new_column_name='muscle_groups')
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
     op.alter_column('gear', 'cost',
-               existing_type=postgresql.DOUBLE_PRECISION(precision=53),
-               nullable=False)
+                    existing_type=postgresql.DOUBLE_PRECISION(precision=53),
+                    nullable=False)
+    op.alter_column('equipment', 'muscle_groups',
+                    new_column_name='categories')
     # ### end Alembic commands ###