From fa8e58f30486995681b9d737d690e3535c5f2ef4 Mon Sep 17 00:00:00 2001 From: Joshua Dirga Date: Sun, 10 Nov 2024 16:26:55 -0500 Subject: [PATCH] Migration: moved scrapers under condition --- .github/workflows/deploy-dev.yml | 3 +- app.py | 85 +++++++++---------- migrations/env.py | 2 +- .../f711f3c11324_initial_migration.py | 12 ++- 4 files changed, 51 insertions(+), 51 deletions(-) diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index a159ac8..3761857 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -39,11 +39,10 @@ jobs: touch tags source tags export IMAGE_TAG=${{ steps.vars.outputs.sha_short }} - echo "export IMAGE_TAG=${IMAGE_TAG}" > tags cd docker-compose docker stack rm the-stack sleep 20s sudo systemctl stop nginx sudo systemctl restart nginx docker stack deploy -c docker-compose.yml the-stack --with-registry-auth - yes | docker system prune -a \ No newline at end of file + yes | docker system prune -a diff --git a/app.py b/app.py index ff0e765..0b2ec16 100644 --- a/app.py +++ b/app.py @@ -9,13 +9,6 @@ from src.database import Base as db from flask_migrate import Migrate from src.schema import Query, Mutation -from src.scrapers.capacities_scraper import fetch_capacities -from src.scrapers.reg_hours_scraper import fetch_reg_building, fetch_reg_facility -from src.scrapers.scraper_helpers import clean_past_hours -from src.scrapers.sp_hours_scraper import fetch_sp_facility -from src.scrapers.equipment_scraper import scrape_equipment -from src.scrapers.class_scraper import fetch_classes -from src.scrapers.activities_scraper import fetch_activity from src.utils.utils import create_gym_table from src.models.openhours import OpenHours from src.database import db_url, db_user, db_password, db_name, db_host, db_port @@ -45,8 +38,6 @@ app.config['SQLALCHEMY_DATABASE_URI'] = db_url app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False -migrate = Migrate(app, db) - schema = Schema(query=Query, mutation=Mutation) swagger = Swagger(app) @@ -72,52 +63,58 @@ def shutdown_session(exception=None): db_session.remove() -# Scrape hours every 15 minutes -@scheduler.task("interval", id="scrape_hours", seconds=900) -def scrape_hours(): - logging.info("Scraping hours from sheets...") - - # Clear hours - db_session.query(OpenHours).delete() +# Create schema.graphql +with open("schema.graphql", "w+") as schema_file: + schema_file.write(schema_printer.print_schema(schema)) + schema_file.close() - fetch_reg_facility() - fetch_reg_building() - fetch_sp_facility() - clean_past_hours() +if __name__ == "__main__": + from src.scrapers.capacities_scraper import fetch_capacities + from src.scrapers.reg_hours_scraper import fetch_reg_building, fetch_reg_facility + from src.scrapers.scraper_helpers import clean_past_hours + from src.scrapers.sp_hours_scraper import fetch_sp_facility + from src.scrapers.equipment_scraper import scrape_equipment + from src.scrapers.class_scraper import fetch_classes + from src.scrapers.activities_scraper import fetch_activity + # Scrape hours every 15 minutes + @scheduler.task("interval", id="scrape_hours", seconds=900) + def scrape_hours(): + logging.info("Scraping hours from sheets...") -# Scrape capacities every 10 minutes -@scheduler.task("interval", id="scrape_capacities", seconds=600) -def scrape_capacities(): - logging.info("Scraping capacities from C2C...") + # Clear hours + db_session.query(OpenHours).delete() - fetch_capacities() + fetch_reg_facility() + fetch_reg_building() + fetch_sp_facility() + clean_past_hours() + # Scrape capacities every 10 minutes -# Scrape classes every hour -@scheduler.task("interval", id="scrape_classes", seconds=3600) -def scrape_classes(): - logging.info("Scraping classes from group-fitness-classes...") + @scheduler.task("interval", id="scrape_capacities", seconds=600) + def scrape_capacities(): + logging.info("Scraping capacities from C2C...") - fetch_classes(10) + fetch_capacities() + # Scrape classes every hour -# Create database and fill it with data -init_db() -create_gym_table() + @scheduler.task("interval", id="scrape_classes", seconds=3600) + def scrape_classes(): + logging.info("Scraping classes from group-fitness-classes...") -scrape_classes() -scrape_hours() -scrape_capacities() -scrape_equipment() + fetch_classes(10) -logging.info("Scraping activities from sheets...") -fetch_activity() + # Create database and fill it with data + init_db() + create_gym_table() -# Create schema.graphql -with open("schema.graphql", "w+") as schema_file: - schema_file.write(schema_printer.print_schema(schema)) - schema_file.close() + scrape_classes() + scrape_hours() + scrape_capacities() + scrape_equipment() + logging.info("Scraping activities from sheets...") + fetch_activity() -if __name__ == "__main__": app.run(host="127.0.0.1", port=5000) diff --git a/migrations/env.py b/migrations/env.py index 169d487..6b3bfa6 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -1,4 +1,5 @@ from __future__ import with_statement +from flask import current_app import logging from logging.config import fileConfig @@ -21,7 +22,6 @@ # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -from flask import current_app config.set_main_option('sqlalchemy.url', current_app.config.get('SQLALCHEMY_DATABASE_URI')) target_metadata = current_app.extensions['migrate'].db.metadata diff --git a/migrations/versions/f711f3c11324_initial_migration.py b/migrations/versions/f711f3c11324_initial_migration.py index 949a2e3..3fc728e 100644 --- a/migrations/versions/f711f3c11324_initial_migration.py +++ b/migrations/versions/f711f3c11324_initial_migration.py @@ -19,14 +19,18 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.alter_column('gear', 'cost', - existing_type=postgresql.DOUBLE_PRECISION(precision=53), - nullable=0) + existing_type=postgresql.DOUBLE_PRECISION(precision=53), + nullable=0) + op.alter_column('equipment', 'categories', + new_column_name='muscle_groups') # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.alter_column('gear', 'cost', - existing_type=postgresql.DOUBLE_PRECISION(precision=53), - nullable=False) + existing_type=postgresql.DOUBLE_PRECISION(precision=53), + nullable=False) + op.alter_column('equipment', 'muscle_groups', + new_column_name='categories') # ### end Alembic commands ###