chore: merge v0.6.0-rc.0 to main

Signed-off-by: Evan Prodromou <[email protected]>

.github/workflows/global-api-develop.yml

@@ -9,9 +9,59 @@ on:
       - k8s/cc-global-api-deploy.yml
       - k8s/cc-global-api.yml
       - k8s/cc-global-api-migrate.yml
+      - .github/workflows/global-api-develop.yml
     branches: ["develop"]
 
 jobs:
+  runTests:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./global-api
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.11
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest pytest-cov
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      - name: Set up database
+        run: |
+          docker run --name github_action_postgresql -d -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_PASSWORD="" postgres
+          sleep 10
+          createuser -w -h localhost -p 5432 -U postgres citycatalyst
+          createdb -w -h localhost -p 5432 -U postgres citycatalyst -O citycatalyst
+          cp env.example .env
+
+      - name: Build coverage file
+        run: |
+          pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=app tests/ | tee pytest-coverage.txt
+
+      - name: Pytest coverage comment
+        uses: MishaKav/pytest-coverage-comment@main
+        with:
+          pytest-coverage-path: ./pytest-coverage.txt
+          junitxml-path: ./pytest.xml
+
+      - name: Upload coverage reports to Codecov
+        continue-on-error: true
+        uses: codecov/[email protected]
+        with:
+          fail_ci_if_error: false
+          flags: unittests
+          name: citycatalyst-global-api
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: Open-Earth-Foundation/CityCatalyst
+
+      - name: Shut down database
+        run: docker stop github_action_postgresql
 
   pushToGHCR:
     runs-on: ubuntu-latest
@@ -55,4 +105,4 @@ jobs:
         run: |
           kubectl create -f k8s/cc-global-api-migrate.yml -n default
           kubectl apply -f k8s/cc-global-api-deploy.yml -n default
-          kubectl rollout restart deployment cc-global-api-deploy -n default
+          kubectl rollout restart deployment cc-global-api-deploy -n default

.github/workflows/global-api-tag.yml

@@ -9,6 +9,7 @@ on:
       - k8s/cc-global-api-deploy.yml
       - k8s/cc-global-api.yml
       - k8s/cc-global-api-migrate.yml
+      - .github/workflows/global-api-tag.yml
     tags:
       - '^v\d+\.\d+\.\d+$'
 

.github/workflows/web-develop.yml

@@ -3,14 +3,20 @@ name: Push Web app to GHCR and deploy to EKS
 on:
   workflow_dispatch:
   push:
+    branches: ["develop"]
     paths:
       - app/**
       - k8s/cc-migrate.yml
       - k8s/cc-web-deploy.yml
       - k8s/cc-web.yml
-      - .github/workflows/web-actions.yml
-    branches: ["develop"]
+      - .github/workflows/web-develop.yml
   pull_request:
+    paths:
+      - app/**
+      - k8s/cc-migrate.yml
+      - k8s/cc-web-deploy.yml
+      - k8s/cc-web.yml
+      - .github/workflows/web-develop.yml
 
 jobs:
   runTests:
@@ -49,22 +55,9 @@ jobs:
       # - name: Run Cypress tests
       #   run: npm run cy:test
 
-      # ... Generate LCOV files or download it from a different job
       - name: Run tests and generate coverage file
         run: npm run ci:test
 
-      # - name: Setup LCOV
-      #   uses: hrishikesh-kadam/setup-lcov@v1
-      # - name: Report code coverage
-      #   uses: zgosalvez/github-actions-report-lcov@v3
-      #   with:
-      #     coverage-files: ./app/lcov*.info
-      #     minimum-coverage: 40
-      #     artifact-name: code-coverage-report
-      #     github-token: ${{ secrets.GITHUB_TOKEN }}
-      #     working-directory: ./app
-      #     update-comment: true
-
       - name: Upload coverage reports to Codecov
         continue-on-error: true
         uses: codecov/[email protected]

.gitignore

@@ -142,3 +142,10 @@ global-api/data-coverage/*.csv
 # Test coverage output from ci:test
 lcov.info
 
+# Data files created by importers
+global-api/importer/IEA_energy/IEA_energy.csv
+global-api/importer/ghgrp_epa/epa.csv
+global-api/importer/mendoza_arg/data/**
+global-api/importer/osm/cache/**
+global-api/importer/osm/data/**
+global-api/data/**

app/migrations/20240308152708-add-cityId-to-userFile.cjs

@@ -0,0 +1,22 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+
+module.exports = {
+  async up (queryInterface, Sequelize) {
+    await queryInterface.addColumn('UserFile', 'city_id', {
+      type: Sequelize.UUID,
+      allowNull: true,
+      references: {
+        model: 'City', 
+        key: 'city_id', 
+      },
+      onUpdate: 'CASCADE',
+      onDelete: 'SET NULL',
+    });
+  },
+
+  async down (queryInterface, Sequelize) {
+    await queryInterface.removeColumn('UserFile', 'city_id');
+  }
+};

app/migrations/20240314122321-add_scope_and_subsector_to_userfile.cjs

@@ -0,0 +1,20 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up (queryInterface, Sequelize) {
+    await queryInterface.addColumn('UserFile', 'scopes', {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+      allowNull: true
+    })
+    await queryInterface.addColumn('UserFile', 'subsectors', {
+      type: Sequelize.ARRAY(Sequelize.STRING),
+      allowNull: true
+    })
+  },
+
+  async down (queryInterface, Sequelize) {
+   await queryInterface.removeColumn('UserFile', 'scopes');
+   await queryInterface.removeColumn('UserFile', 'subsectors')
+  }
+};

app/migrations/20240318105130-region-population.cjs

@@ -0,0 +1,16 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    await queryInterface.addColumn(
+      "Population",
+      "region_population",
+      Sequelize.BIGINT,
+    );
+  },
+
+  async down(queryInterface) {
+    await queryInterface.removeColumn("Population", "region_population");
+  },
+};

app/migrations/20240329160302-new-catalog.cjs

@@ -0,0 +1,79 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    return queryInterface.sequelize.transaction(async (transaction) => {
+      await queryInterface.addColumn("DataSource", "methodology_description", {
+        type: Sequelize.TEXT,
+        transaction,
+      });
+      await queryInterface.addColumn(
+        "DataSource",
+        "transformation_description",
+        {
+          type: Sequelize.TEXT,
+          transaction,
+        },
+      );
+      await queryInterface.addColumn("DataSource", "dataset_name", {
+        type: Sequelize.TEXT,
+        transaction,
+      });
+      await queryInterface.renameColumn(
+        "DataSource",
+        "name",
+        "datasource_name",
+        {
+          transaction,
+        },
+      );
+      await queryInterface.renameColumn(
+        "DataSource",
+        "description",
+        "dataset_description",
+        {
+          transaction,
+        },
+      );
+    });
+  },
+
+  async down(queryInterface) {
+    return queryInterface.sequelize.transaction(async (transaction) => {
+      await queryInterface.removeColumn(
+        "DataSource",
+        "methodology_description",
+        {
+          transaction,
+        },
+      );
+      await queryInterface.removeColumn(
+        "DataSource",
+        "transformation_description",
+        {
+          transaction,
+        },
+      );
+      await queryInterface.removeColumn("DataSource", "dataset_name", {
+        transaction,
+      });
+      await queryInterface.renameColumn(
+        "DataSource",
+        "datasource_name",
+        "name",
+        {
+          transaction,
+        },
+      );
+      await queryInterface.renameColumn(
+        "DataSource",
+        "dataset_description",
+        "description",
+        {
+          transaction,
+        },
+      );
+    });
+  },
+};