From f9f628e96f1ab61a6999243f1288d5bdee4f0e6c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 07:48:41 +0000 Subject: [PATCH 01/51] chore(deps): bump @types/jsonwebtoken from 9.0.4 to 9.0.6 in /app Bumps [@types/jsonwebtoken](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/jsonwebtoken) from 9.0.4 to 9.0.6. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/jsonwebtoken) --- updated-dependencies: - dependency-name: "@types/jsonwebtoken" dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- app/package-lock.json | 8 ++++---- app/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/package-lock.json b/app/package-lock.json index 60f16664b..07970c1b4 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -21,7 +21,7 @@ "@types/geojson": "^7946.0.14", "@types/http-errors": "^2.0.4", "@types/js-cookie": "^3.0.6", - "@types/jsonwebtoken": "^9.0.4", + "@types/jsonwebtoken": "^9.0.6", "@types/node": "20.11.5", "@types/nodemailer": "^6.4.14", "@types/pg": "^8.11.4", @@ -9174,9 +9174,9 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==" }, "node_modules/@types/jsonwebtoken": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.4.tgz", - "integrity": "sha512-8UYapdmR0QlxgvJmyE8lP7guxD0UGVMfknsdtCFZh4ovShdBl3iOI4zdvqBHrB/IS+xUj3PSx73Qkey1fhWz+g==", + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.6.tgz", + "integrity": "sha512-/5hndP5dCjloafCXns6SZyESp3Ldq7YjH3zwzwczYnjxIT0Fqzk5ROSYVGfFyczIue7IUEj8hkvLbPoLQ18vQw==", "dependencies": { "@types/node": "*" } diff --git a/app/package.json b/app/package.json index 3a646576b..652937bd5 100644 --- a/app/package.json +++ b/app/package.json @@ -43,7 +43,7 @@ "@types/geojson": "^7946.0.14", "@types/http-errors": "^2.0.4", "@types/js-cookie": "^3.0.6", - "@types/jsonwebtoken": "^9.0.4", + "@types/jsonwebtoken": "^9.0.6", "@types/node": "20.11.5", "@types/nodemailer": "^6.4.14", "@types/pg": "^8.11.4", From d09ac2eb4f7d4f251d59db39fc053e199b0e6a1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 07:49:08 +0000 Subject: [PATCH 02/51] chore(deps): bump react-redux from 8.1.3 to 9.1.0 in /app Bumps [react-redux](https://github.com/reduxjs/react-redux) from 8.1.3 to 9.1.0. - [Release notes](https://github.com/reduxjs/react-redux/releases) - [Changelog](https://github.com/reduxjs/react-redux/blob/master/CHANGELOG.md) - [Commits](https://github.com/reduxjs/react-redux/compare/v8.1.3...v9.1.0) --- updated-dependencies: - dependency-name: react-redux dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- app/package-lock.json | 77 ++++++++++++++++--------------------------- app/package.json | 2 +- 2 files changed, 30 insertions(+), 49 deletions(-) diff --git a/app/package-lock.json b/app/package-lock.json index 60f16664b..868512251 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -65,7 +65,7 @@ "react-i18next": "^14.0.5", "react-icons": "^4.11.0", "react-intersection-observer": "^9.8.1", - "react-redux": "^8.1.3", + "react-redux": "^9.1.0", "redux-persist": "^6.0.0", "sequelize": "^6.37.1", "sequelize-cli": "^6.6.2", @@ -5880,6 +5880,22 @@ } } }, + "node_modules/@reduxjs/toolkit/node_modules/redux": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/redux/-/redux-4.2.1.tgz", + "integrity": "sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==", + "dependencies": { + "@babel/runtime": "^7.9.2" + } + }, + "node_modules/@reduxjs/toolkit/node_modules/redux-thunk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.4.2.tgz", + "integrity": "sha512-+P3TjtnP0k/FEjcBL5FZpoovtvrTNT/UXd4/sluaSyrURlSlhLSzEdfsTBW7WsKB6yPvgd7q/iZPICFjW4o57Q==", + "peerDependencies": { + "redux": "^4" + } + }, "node_modules/@rushstack/eslint-patch": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.3.3.tgz", @@ -9114,15 +9130,6 @@ "@types/node": "*" } }, - "node_modules/@types/hoist-non-react-statics": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/@types/hoist-non-react-statics/-/hoist-non-react-statics-3.3.1.tgz", - "integrity": "sha512-iMIqiko6ooLrTh1joXodJK5X9xeEALT1kM5G3ZLhD3hszxBdIEd5C75U834D9mLcINgD4OyZf5uQXjkuYydWvA==", - "dependencies": { - "@types/react": "*", - "hoist-non-react-statics": "^3.3.0" - } - }, "node_modules/@types/html-minifier-terser": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/@types/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz", @@ -16807,6 +16814,7 @@ "version": "3.3.2", "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", + "peer": true, "dependencies": { "react-is": "^16.7.0" } @@ -22507,35 +22515,23 @@ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" }, "node_modules/react-redux": { - "version": "8.1.3", - "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-8.1.3.tgz", - "integrity": "sha512-n0ZrutD7DaX/j9VscF+uTALI3oUPa/pO4Z3soOBIjuRn/FzVu6aehhysxZCLi6y7duMf52WNZGMl7CtuK5EnRw==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.1.0.tgz", + "integrity": "sha512-6qoDzIO+gbrza8h3hjMA9aq4nwVFCKFtY2iLxCtVT38Swyy2C/dJCGBXHeHLtx6qlg/8qzc2MrhOeduf5K32wQ==", "dependencies": { - "@babel/runtime": "^7.12.1", - "@types/hoist-non-react-statics": "^3.3.1", "@types/use-sync-external-store": "^0.0.3", - "hoist-non-react-statics": "^3.3.2", - "react-is": "^18.0.0", "use-sync-external-store": "^1.0.0" }, "peerDependencies": { - "@types/react": "^16.8 || ^17.0 || ^18.0", - "@types/react-dom": "^16.8 || ^17.0 || ^18.0", - "react": "^16.8 || ^17.0 || ^18.0", - "react-dom": "^16.8 || ^17.0 || ^18.0", - "react-native": ">=0.59", - "redux": "^4 || ^5.0.0-beta.0" + "@types/react": "^18.2.25", + "react": "^18.0", + "react-native": ">=0.69", + "redux": "^5.0.0" }, "peerDependenciesMeta": { "@types/react": { "optional": true }, - "@types/react-dom": { - "optional": true - }, - "react-dom": { - "optional": true - }, "react-native": { "optional": true }, @@ -22544,11 +22540,6 @@ } } }, - "node_modules/react-redux/node_modules/react-is": { - "version": "18.2.0", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", - "integrity": "sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==" - }, "node_modules/react-refresh": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.14.0.tgz", @@ -22869,12 +22860,10 @@ } }, "node_modules/redux": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/redux/-/redux-4.2.1.tgz", - "integrity": "sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==", - "dependencies": { - "@babel/runtime": "^7.9.2" - } + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", + "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", + "peer": true }, "node_modules/redux-persist": { "version": "6.0.0", @@ -22884,14 +22873,6 @@ "redux": ">4.0.0" } }, - "node_modules/redux-thunk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.4.2.tgz", - "integrity": "sha512-+P3TjtnP0k/FEjcBL5FZpoovtvrTNT/UXd4/sluaSyrURlSlhLSzEdfsTBW7WsKB6yPvgd7q/iZPICFjW4o57Q==", - "peerDependencies": { - "redux": "^4" - } - }, "node_modules/reflect.getprototypeof": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz", diff --git a/app/package.json b/app/package.json index 3a646576b..b948b1181 100644 --- a/app/package.json +++ b/app/package.json @@ -87,7 +87,7 @@ "react-i18next": "^14.0.5", "react-icons": "^4.11.0", "react-intersection-observer": "^9.8.1", - "react-redux": "^8.1.3", + "react-redux": "^9.1.0", "redux-persist": "^6.0.0", "sequelize": "^6.37.1", "sequelize-cli": "^6.6.2", From 3be3170ec7bb15d0efbdfa51d5f96eb03341fdf1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 07:49:40 +0000 Subject: [PATCH 03/51] chore(deps): bump tailwindcss from 3.4.1 to 3.4.3 in /app Bumps [tailwindcss](https://github.com/tailwindlabs/tailwindcss) from 3.4.1 to 3.4.3. - [Release notes](https://github.com/tailwindlabs/tailwindcss/releases) - [Changelog](https://github.com/tailwindlabs/tailwindcss/blob/v3.4.3/CHANGELOG.md) - [Commits](https://github.com/tailwindlabs/tailwindcss/compare/v3.4.1...v3.4.3) --- updated-dependencies: - dependency-name: tailwindcss dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- app/package-lock.json | 16 ++++++++-------- app/package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/app/package-lock.json b/app/package-lock.json index 60f16664b..9d8347591 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -69,7 +69,7 @@ "redux-persist": "^6.0.0", "sequelize": "^6.37.1", "sequelize-cli": "^6.6.2", - "tailwindcss": "3.4.1", + "tailwindcss": "3.4.3", "tsx": "^4.7.0", "typescript": "5.3.3", "uuid": "^9.0.1", @@ -17971,9 +17971,9 @@ } }, "node_modules/jiti": { - "version": "1.20.0", - "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.20.0.tgz", - "integrity": "sha512-3TV69ZbrvV6U5DfQimop50jE9Dl6J8O1ja1dvBbMba/sZ3YBEQqJ2VZRoQPVnhlzjNtU1vaXRZVrVjU4qtm8yA==", + "version": "1.21.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.0.tgz", + "integrity": "sha512-gFqAIbuKyyso/3G2qhiO2OM6shY6EPP/R0+mkDbyspxKazh8BXDC5FiFsUjlczgdNz/vfra0da2y+aHrusLG/Q==", "bin": { "jiti": "bin/jiti.js" } @@ -25443,9 +25443,9 @@ } }, "node_modules/tailwindcss": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.1.tgz", - "integrity": "sha512-qAYmXRfk3ENzuPBakNK0SRrUDipP8NQnEY6772uDhflcQz5EhRdD7JNZxyrFHVQNCwULPBn6FNPp9brpO7ctcA==", + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.3.tgz", + "integrity": "sha512-U7sxQk/n397Bmx4JHbJx/iSOOv5G+II3f1kpLpY2QeUv5DcPdcTsYLlusZfq1NthHS1c1cZoyFmmkex1rzke0A==", "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -25455,7 +25455,7 @@ "fast-glob": "^3.3.0", "glob-parent": "^6.0.2", "is-glob": "^4.0.3", - "jiti": "^1.19.1", + "jiti": "^1.21.0", "lilconfig": "^2.1.0", "micromatch": "^4.0.5", "normalize-path": "^3.0.0", diff --git a/app/package.json b/app/package.json index 3a646576b..a657a5f39 100644 --- a/app/package.json +++ b/app/package.json @@ -91,7 +91,7 @@ "redux-persist": "^6.0.0", "sequelize": "^6.37.1", "sequelize-cli": "^6.6.2", - "tailwindcss": "3.4.1", + "tailwindcss": "3.4.3", "tsx": "^4.7.0", "typescript": "5.3.3", "uuid": "^9.0.1", From 1640b83323c8c10bc867685cc82436dc5fabc853 Mon Sep 17 00:00:00 2001 From: Milan Gruner Date: Wed, 3 Apr 2024 14:59:44 +0200 Subject: [PATCH 04/51] fix(ui): disconnecting source progress indicator being enabled for all sources --- .../[lng]/[inventory]/data/[step]/page.tsx | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/app/src/app/[lng]/[inventory]/data/[step]/page.tsx b/app/src/app/[lng]/[inventory]/data/[step]/page.tsx index 2ac9c5b0b..92fd3d47f 100644 --- a/app/src/app/[lng]/[inventory]/data/[step]/page.tsx +++ b/app/src/app/[lng]/[inventory]/data/[step]/page.tsx @@ -479,6 +479,9 @@ export default function AddDataSteps({ {}, ); + const [disconnectingDataSourceId, setDisconnectingDataSourceId] = useState< + string | null + >(null); const [disconnectThirdPartyData, { isLoading: isDisconnectLoading }] = api.useDisconnectThirdPartyDataMutation(); @@ -486,17 +489,20 @@ export default function AddDataSteps({ source: DataSourceWithRelations, ) => { if (isSourceConnected(source)) { - source.inventoryValues!.forEach( - async (inventoryValue: InventoryValueAttributes) => { - await disconnectThirdPartyData({ - inventoryId: inventoryValue.inventoryId, - subCategoryId: inventoryValue.subCategoryId, - }).then((res: any) => { - // Todo show alert - onSearchDataSourcesClicked(); - }); - }, + setDisconnectingDataSourceId(source.datasourceId); + await Promise.all( + source.inventoryValues!.map( + async (inventoryValue: InventoryValueAttributes) => { + return await disconnectThirdPartyData({ + inventoryId: inventoryValue.inventoryId, + subCategoryId: inventoryValue.subCategoryId, + }); + }, + ), ); + // TODO show alert + setDisconnectingDataSourceId(null); + onSearchDataSourcesClicked(); } else { console.log("Something went wrong"); } @@ -747,7 +753,10 @@ export default function AddDataSteps({ px={6} py={4} onClick={() => onDisconnectThirdPartyData(source)} - isLoading={isDisconnectLoading} + isLoading={ + isDisconnectLoading && + source.datasourceId === disconnectingDataSourceId + } onMouseEnter={() => onButtonHover(source)} onMouseLeave={() => onMouseLeave(source)} leftIcon={} From 988f3f70878648ed9ef57a0f7abb6a9b293cc8f1 Mon Sep 17 00:00:00 2001 From: mfonsecaOEF Date: Wed, 3 Apr 2024 14:19:43 -0600 Subject: [PATCH 05/51] fix:adding_missing_columns --- .../949c5b9cc18d_ef_in_country_code_table.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 global-api/migrations/versions/949c5b9cc18d_ef_in_country_code_table.py diff --git a/global-api/migrations/versions/949c5b9cc18d_ef_in_country_code_table.py b/global-api/migrations/versions/949c5b9cc18d_ef_in_country_code_table.py new file mode 100644 index 000000000..0dfcbf6cd --- /dev/null +++ b/global-api/migrations/versions/949c5b9cc18d_ef_in_country_code_table.py @@ -0,0 +1,28 @@ +"""EF_in_country_code_table + +Revision ID: 949c5b9cc18d +Revises: 583858ff1aa8 +Create Date: 2024-04-03 10:24:31.131245 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '949c5b9cc18d' +down_revision: Union[str, None] = '583858ff1aa8' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column("country_code", "emission_factor_value", nullable=True, type_=sa.Float) + op.alter_column("country_code", "emission_factor_units", nullable=True, type_=sa.String) + + +def downgrade() -> None: + op.drop_column("country_code", "emission_factor_value", nullable=False, type_=sa.Float) + op.drop_column("country_code", "emission_factor_units", nullable=False, type_=sa.String) From c9527ae4c932483a97cbc0279410e783c99e072d Mon Sep 17 00:00:00 2001 From: Milan Gruner Date: Fri, 5 Apr 2024 11:51:58 +0200 Subject: [PATCH 06/51] fix(ci): add quotes to new env vars in web-develop workflow --- .github/workflows/web-develop.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/web-develop.yml b/.github/workflows/web-develop.yml index 6bb3a1623..1c62d5ad6 100644 --- a/.github/workflows/web-develop.yml +++ b/.github/workflows/web-develop.yml @@ -127,8 +127,8 @@ jobs: kubectl set env deployment/cc-web-deploy CHAT_PROVIDER=openai kubectl set env deployment/cc-web-deploy OPENAI_API_KEY=${{secrets.OPENAI_API_KEY}} kubectl set env deployment/cc-web-deploy HUGGINGFACE_API_KEY=${{secrets.HUGGINGFACE_API_KEY}} - kubectl set env deployment/cc-web-deploy ADMIN_EMAILS=${{secrets.ADMIN_EMAILS}} - kubectl set env deployment/cc-web-deploy ADMIN_NAMES=${{secrets.ADMIN_NAMES}} + kubectl set env deployment/cc-web-deploy "ADMIN_EMAILS=${{secrets.ADMIN_EMAILS}}" + kubectl set env deployment/cc-web-deploy "ADMIN_NAMES=${{secrets.ADMIN_NAMES}}" kubectl set env deployment/cc-web-deploy "DEFAULT_ADMIN_EMAIL=${{secrets.DEFAULT_ADMIN_EMAIL}}" kubectl set env deployment/cc-web-deploy "DEFAULT_ADMIN_PASSWORD=${{secrets.DEFAULT_ADMIN_PASSWORD}}" kubectl create -f k8s/cc-create-admin.yml -n default From a3327fb9088756b973e0a32fd6f46adc0fa42293 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 08:01:10 +0000 Subject: [PATCH 07/51] chore(deps): bump osmnx from 1.9.1 to 1.9.2 in /global-api Bumps [osmnx](https://github.com/gboeing/osmnx) from 1.9.1 to 1.9.2. - [Changelog](https://github.com/gboeing/osmnx/blob/main/CHANGELOG.md) - [Commits](https://github.com/gboeing/osmnx/compare/v1.9.1...v1.9.2) --- updated-dependencies: - dependency-name: osmnx dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- global-api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global-api/requirements.txt b/global-api/requirements.txt index 9440b4bce..b88c8868c 100644 --- a/global-api/requirements.txt +++ b/global-api/requirements.txt @@ -6,7 +6,7 @@ flake8==7.0.0 fsspec==2024.* geopandas==0.14.3 mypy==1.9.0 -osmnx==1.9.1 +osmnx==1.9.2 pandas==2.2.1 psycopg2-binary==2.9.9 pydantic-settings==2.* From 2ee4500fe4b92d227479fd852eb00bf1519b49d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 08:01:32 +0000 Subject: [PATCH 08/51] chore(deps): update scipy requirement in /global-api Updates the requirements on [scipy](https://github.com/scipy/scipy) to permit the latest version. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.12.0rc1...v1.13.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- global-api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global-api/requirements.txt b/global-api/requirements.txt index 9440b4bce..f423ef099 100644 --- a/global-api/requirements.txt +++ b/global-api/requirements.txt @@ -12,7 +12,7 @@ psycopg2-binary==2.9.9 pydantic-settings==2.* pytest==8.1.1 rioxarray==0.15.* -scipy==1.12.* +scipy==1.13.* shapely==2.0.3 SQLAlchemy==2.0.29 tqdm==4.66.* From 96f9f9bb69900ab2e2f1029b1959a2936bb6dfb3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 08:01:48 +0000 Subject: [PATCH 09/51] chore(deps): bump fastapi from 0.110.0 to 0.110.1 in /global-api Bumps [fastapi](https://github.com/tiangolo/fastapi) from 0.110.0 to 0.110.1. - [Release notes](https://github.com/tiangolo/fastapi/releases) - [Commits](https://github.com/tiangolo/fastapi/compare/0.110.0...0.110.1) --- updated-dependencies: - dependency-name: fastapi dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- global-api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global-api/requirements.txt b/global-api/requirements.txt index 9440b4bce..616164247 100644 --- a/global-api/requirements.txt +++ b/global-api/requirements.txt @@ -1,7 +1,7 @@ aiohttp==3.9.* alembic==1.13.1 black==24.3.0 -fastapi==0.110.0 +fastapi==0.110.1 flake8==7.0.0 fsspec==2024.* geopandas==0.14.3 From 02fa7d4ccd1f627ccd7f6708b10902eb15520eba Mon Sep 17 00:00:00 2001 From: Milan Gruner Date: Mon, 8 Apr 2024 13:31:41 +0200 Subject: [PATCH 10/51] feat(ui): implement optional validation for PasswordInput and use on signup --- app/src/app/[lng]/auth/signup/page.tsx | 13 +++++++++++-- app/src/components/password-input.tsx | 9 +++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/app/src/app/[lng]/auth/signup/page.tsx b/app/src/app/[lng]/auth/signup/page.tsx index 9ad48f8f9..4c0f6e156 100644 --- a/app/src/app/[lng]/auth/signup/page.tsx +++ b/app/src/app/[lng]/auth/signup/page.tsx @@ -85,7 +85,11 @@ export default function Signup({ if (!res.ok) { const data = await res.json(); logger.error("Failed to sign up", data); - setError(data.error.message); + let message = data.error.message; + if (message === "Entity exists already.") { + message = t("user-exists-already"); + } + setError(message); return; } @@ -134,7 +138,12 @@ export default function Signup({ - + setShowPassword(!showPassword); @@ -42,6 +45,12 @@ export default function PasswordInput({ {...register(id, { required: t("password-required"), minLength: { value: 4, message: t("min-length", { length: 4 }) }, + pattern: shouldValidate + ? { + value: passwordRegex, + message: t("password-invalid"), + } + : undefined, })} /> From 4221abcb726f9f377984ee3c2aadef6784e70e29 Mon Sep 17 00:00:00 2001 From: Milan Gruner Date: Mon, 8 Apr 2024 13:32:10 +0200 Subject: [PATCH 11/51] fix(ui): add new i18n strings for signup errors, remove duplicate cancel entry --- app/src/i18n/locales/de/auth.json | 5 +++-- app/src/i18n/locales/en/auth.json | 5 +++-- app/src/i18n/locales/es/auth.json | 7 ++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/app/src/i18n/locales/de/auth.json b/app/src/i18n/locales/de/auth.json index 23569017a..58a732685 100644 --- a/app/src/i18n/locales/de/auth.json +++ b/app/src/i18n/locales/de/auth.json @@ -30,6 +30,8 @@ "accept-terms-required": "Bitte akzeptieren Sie die Geschäftsbedingungen, um sich zu registrieren", "create-account": "Registrieren", "have-account": "Haben Sie bereits ein Konto?", + "password-invalid": "Das Passwort entspricht nicht den Anforderungen", + "user-exists-already": "Ein Benutzer existiert bereits mit dieser E-Mail-Adresse", "forgot-password-heading": "Passwort vergessen?", "forgot-password-details": "Geben Sie die Email Adresse die sie für die Registrierung nutzten ein und wir senden Ihnen Instruktionen zur Zurücksetzung zu.", @@ -52,6 +54,5 @@ "update-password-heading": "Aktualisieren Sie ihr Passwort", "update-password-details": "Aktualisieren Sie ihr Passwort. Benutzen Sie dieses von nun an.", - "reset-button": "Password zurücksetzen", - "cancel": "Abbrechen" + "reset-button": "Password zurücksetzen" } diff --git a/app/src/i18n/locales/en/auth.json b/app/src/i18n/locales/en/auth.json index acd771380..36926ba15 100644 --- a/app/src/i18n/locales/en/auth.json +++ b/app/src/i18n/locales/en/auth.json @@ -30,6 +30,8 @@ "accept-terms-required": "Please accept the terms and conditions in order to sign up", "create-account": "Create Account", "have-account": "Already have an account?", + "password-invalid": "Password does not meet the requirements", + "user-exists-already": "User with this email already exists", "forgot-password-heading": "Forgot Password?", "forgot-password-details": "Enter the email address you used when you joined and we'll send you instructions to reset your password.", @@ -52,6 +54,5 @@ "update-password-heading": "Update your Password", "update-password-details": "Update your password. This is the one that you will use from now on.", - "reset-button": "Reset Password", - "cancel": "Cancel" + "reset-button": "Reset Password" } diff --git a/app/src/i18n/locales/es/auth.json b/app/src/i18n/locales/es/auth.json index b6e98fb9d..21411ecbe 100644 --- a/app/src/i18n/locales/es/auth.json +++ b/app/src/i18n/locales/es/auth.json @@ -30,6 +30,8 @@ "accept-terms-required": "Acepta los términos y condiciones para registrarse", "create-account": "Crear cuenta", "have-account": "¿Ya tienes una cuenta?", + "password-invalid": "La contraseña no cumple con los requisitos", + "user-exists-already": "El usuario con este correo electrónico ya existe", "forgot-password-heading": "¿Olvidaste tu contraseña?", "forgot-password-details": "Ingresa la dirección de correo electrónico que utilizaste cuando te registraste y te enviaremos instrucciones para restablecer tu contraseña.", @@ -52,6 +54,5 @@ "update-password-heading": "Actualiza tu Contraseña", "update-password-details": "Actualiza tu contraseña. Esta es la que usarás a partir de ahora.", - "reset-button": "Restablecer contraseña", - "cancel": "Cancelar" -} + "reset-button": "Restablecer contraseña" +} From 14608e5cd6f45b3a574c3dccea259fc1f1d3d883 Mon Sep 17 00:00:00 2001 From: Milan Gruner Date: Mon, 8 Apr 2024 13:32:35 +0200 Subject: [PATCH 12/51] fix(ui): no null check for file.subsectors on add data page --- app/src/app/[lng]/[inventory]/data/[step]/page.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/src/app/[lng]/[inventory]/data/[step]/page.tsx b/app/src/app/[lng]/[inventory]/data/[step]/page.tsx index 2ac9c5b0b..1cb2c1c95 100644 --- a/app/src/app/[lng]/[inventory]/data/[step]/page.tsx +++ b/app/src/app/[lng]/[inventory]/data/[step]/page.tsx @@ -304,8 +304,7 @@ export default function AddDataSteps({ const dataSources = allDataSources?.filter(({ source, data }) => { const referenceNumber = source.subCategory?.referenceNumber || source.subSector?.referenceNumber; - if (!data) return false; - if (!referenceNumber) return false; + if (!data || !referenceNumber) return false; const sectorReferenceNumber = referenceNumber.split(".")[0]; return sectorReferenceNumber === currentStep.referenceNumber; @@ -946,7 +945,7 @@ export default function AddDataSteps({ - {file.subsectors.split(",").map((item: any) => ( + {file.subsectors?.split(",").map((item: any) => ( Date: Mon, 8 Apr 2024 11:39:09 +0000 Subject: [PATCH 13/51] chore(deps): bump pg from 8.11.3 to 8.11.5 in /app Bumps [pg](https://github.com/brianc/node-postgres/tree/HEAD/packages/pg) from 8.11.3 to 8.11.5. - [Changelog](https://github.com/brianc/node-postgres/blob/master/CHANGELOG.md) - [Commits](https://github.com/brianc/node-postgres/commits/pg@8.11.5/packages/pg) --- updated-dependencies: - dependency-name: pg dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- app/package-lock.json | 49 +++++++++++++++---------------------------- app/package.json | 2 +- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/app/package-lock.json b/app/package-lock.json index 1ec3662d2..6fc758580 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -52,7 +52,7 @@ "next-auth": "^4.24.5", "nodemailer": "^6.9.13", "openai": "^4.28.0", - "pg": "^8.11.3", + "pg": "^8.11.5", "pg-hstore": "^2.3.4", "pigeon-maps": "^0.21.3", "pino": "^8.16.2", @@ -11653,14 +11653,6 @@ "node": ">=0.10" } }, - "node_modules/buffer-writer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", - "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", - "engines": { - "node": ">=4" - } - }, "node_modules/buffer-xor": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", @@ -20079,11 +20071,6 @@ "node": ">=6" } }, - "node_modules/packet-reader": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", - "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" - }, "node_modules/pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", @@ -20318,15 +20305,13 @@ "peer": true }, "node_modules/pg": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", - "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", - "dependencies": { - "buffer-writer": "2.0.0", - "packet-reader": "1.0.0", - "pg-connection-string": "^2.6.2", - "pg-pool": "^3.6.1", - "pg-protocol": "^1.6.0", + "version": "8.11.5", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.5.tgz", + "integrity": "sha512-jqgNHSKL5cbDjFlHyYsCXmQDrfIX/3RsNwYqpd4N0Kt8niLuNoRNH+aazv6cOd43gPh9Y4DjQCtb+X0MH0Hvnw==", + "dependencies": { + "pg-connection-string": "^2.6.4", + "pg-pool": "^3.6.2", + "pg-protocol": "^1.6.1", "pg-types": "^2.1.0", "pgpass": "1.x" }, @@ -20352,9 +20337,9 @@ "optional": true }, "node_modules/pg-connection-string": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", - "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" + "version": "2.6.4", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", + "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" }, "node_modules/pg-hstore": { "version": "2.3.4", @@ -20384,17 +20369,17 @@ } }, "node_modules/pg-pool": { - "version": "3.6.1", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", - "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", + "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", "peerDependencies": { "pg": ">=8.0" } }, "node_modules/pg-protocol": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", - "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", + "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==" }, "node_modules/pg-types": { "version": "4.0.1", diff --git a/app/package.json b/app/package.json index ac2b0cea5..cc9e3248a 100644 --- a/app/package.json +++ b/app/package.json @@ -74,7 +74,7 @@ "next-auth": "^4.24.5", "nodemailer": "^6.9.13", "openai": "^4.28.0", - "pg": "^8.11.3", + "pg": "^8.11.5", "pg-hstore": "^2.3.4", "pigeon-maps": "^0.21.3", "pino": "^8.16.2", From 2dfe9df369e513b68f40bdf718b96b017626fe70 Mon Sep 17 00:00:00 2001 From: Milan Gruner Date: Mon, 8 Apr 2024 13:46:47 +0200 Subject: [PATCH 14/51] fix(deps): also upgrade @reduxjs/tooolkit to 2.2.3 --- app/package-lock.json | 57 ++++++++++++++++++------------------------- app/package.json | 2 +- 2 files changed, 25 insertions(+), 34 deletions(-) diff --git a/app/package-lock.json b/app/package-lock.json index 868512251..0df448b3a 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -13,7 +13,7 @@ "@chakra-ui/react": "^2.8.2", "@huggingface/inference": "^2.6.4", "@react-email/components": "^0.0.11", - "@reduxjs/toolkit": "^1.9.7", + "@reduxjs/toolkit": "^2.2.3", "@storybook/cli": "^7.6.12", "@storybook/react": "^7.4.5", "@storybook/testing-library": "^0.2.2", @@ -5858,18 +5858,18 @@ } }, "node_modules/@reduxjs/toolkit": { - "version": "1.9.7", - "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-1.9.7.tgz", - "integrity": "sha512-t7v8ZPxhhKgOKtU+uyJT13lu4vL7az5aFi4IdoDs/eS548edn2M8Ik9h8fxgvMjGoAUVFSt6ZC1P5cWmQ014QQ==", + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.2.3.tgz", + "integrity": "sha512-76dll9EnJXg4EVcI5YNxZA/9hSAmZsFqzMmNRHvIlzw2WS/twfcVX3ysYrWGJMClwEmChQFC4yRq74tn6fdzRA==", "dependencies": { - "immer": "^9.0.21", - "redux": "^4.2.1", - "redux-thunk": "^2.4.2", - "reselect": "^4.1.8" + "immer": "^10.0.3", + "redux": "^5.0.1", + "redux-thunk": "^3.1.0", + "reselect": "^5.0.1" }, "peerDependencies": { "react": "^16.9.0 || ^17.0.0 || ^18", - "react-redux": "^7.2.1 || ^8.0.2" + "react-redux": "^7.2.1 || ^8.1.3 || ^9.0.0" }, "peerDependenciesMeta": { "react": { @@ -5880,22 +5880,6 @@ } } }, - "node_modules/@reduxjs/toolkit/node_modules/redux": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/redux/-/redux-4.2.1.tgz", - "integrity": "sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==", - "dependencies": { - "@babel/runtime": "^7.9.2" - } - }, - "node_modules/@reduxjs/toolkit/node_modules/redux-thunk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-2.4.2.tgz", - "integrity": "sha512-+P3TjtnP0k/FEjcBL5FZpoovtvrTNT/UXd4/sluaSyrURlSlhLSzEdfsTBW7WsKB6yPvgd7q/iZPICFjW4o57Q==", - "peerDependencies": { - "redux": "^4" - } - }, "node_modules/@rushstack/eslint-patch": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.3.3.tgz", @@ -17127,9 +17111,9 @@ "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==" }, "node_modules/immer": { - "version": "9.0.21", - "resolved": "https://registry.npmjs.org/immer/-/immer-9.0.21.tgz", - "integrity": "sha512-bc4NBHqOqSfRW7POMkHd51LvClaeMXpm8dx0e8oE2GORbq5aRK7Bxl4FyzVLdGtLmvLKL7BTDBG5ACQm4HWjTA==", + "version": "10.0.4", + "resolved": "https://registry.npmjs.org/immer/-/immer-10.0.4.tgz", + "integrity": "sha512-cuBuGK40P/sk5IzWa9QPUaAdvPHjkk1c+xYsd9oZw+YQQEV+10G0P5uMpGctZZKnyQ+ibRO08bD25nWLmYi2pw==", "funding": { "type": "opencollective", "url": "https://opencollective.com/immer" @@ -22862,8 +22846,7 @@ "node_modules/redux": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", - "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", - "peer": true + "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==" }, "node_modules/redux-persist": { "version": "6.0.0", @@ -22873,6 +22856,14 @@ "redux": ">4.0.0" } }, + "node_modules/redux-thunk": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-3.1.0.tgz", + "integrity": "sha512-NW2r5T6ksUKXCabzhL9z+h206HQw/NJkcLm1GPImRQ8IzfXwRGqjVhKJGauHirT0DAuyy6hjdnMZaRoAcy0Klw==", + "peerDependencies": { + "redux": "^5.0.0" + } + }, "node_modules/reflect.getprototypeof": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz", @@ -23144,9 +23135,9 @@ "dev": true }, "node_modules/reselect": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/reselect/-/reselect-4.1.8.tgz", - "integrity": "sha512-ab9EmR80F/zQTMNeneUr4cv+jSwPJgIlvEmVwLerwrWVbpLlBuls9XHzIeTFy4cegU2NHBp3va0LKOzU5qFEYQ==" + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/reselect/-/reselect-5.1.0.tgz", + "integrity": "sha512-aw7jcGLDpSgNDyWBQLv2cedml85qd95/iszJjN988zX1t7AVRJi19d9kto5+W7oCfQ94gyo40dVbT6g2k4/kXg==" }, "node_modules/reserved-words": { "version": "0.1.2", diff --git a/app/package.json b/app/package.json index b948b1181..40b6c1d37 100644 --- a/app/package.json +++ b/app/package.json @@ -35,7 +35,7 @@ "@chakra-ui/react": "^2.8.2", "@huggingface/inference": "^2.6.4", "@react-email/components": "^0.0.11", - "@reduxjs/toolkit": "^1.9.7", + "@reduxjs/toolkit": "^2.2.3", "@storybook/cli": "^7.6.12", "@storybook/react": "^7.4.5", "@storybook/testing-library": "^0.2.2", From 5a871da29715f1dbab8c5db9cc9e4c99d1564ca7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 12:18:05 +0000 Subject: [PATCH 15/51] chore(deps): bump openai from 4.28.0 to 4.33.0 in /app Bumps [openai](https://github.com/openai/openai-node) from 4.28.0 to 4.33.0. - [Release notes](https://github.com/openai/openai-node/releases) - [Changelog](https://github.com/openai/openai-node/blob/master/CHANGELOG.md) - [Commits](https://github.com/openai/openai-node/compare/v4.28.0...v4.33.0) --- updated-dependencies: - dependency-name: openai dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- app/package-lock.json | 49 ++++--------------------------------------- app/package.json | 2 +- 2 files changed, 5 insertions(+), 46 deletions(-) diff --git a/app/package-lock.json b/app/package-lock.json index 9a094999f..80d07a922 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -51,7 +51,7 @@ "next": "14.1.3", "next-auth": "^4.24.5", "nodemailer": "^6.9.13", - "openai": "^4.28.0", + "openai": "^4.33.0", "pg": "^8.11.5", "pg-hstore": "^2.3.4", "pigeon-maps": "^0.21.3", @@ -11226,11 +11226,6 @@ "bare-os": "^2.1.0" } }, - "node_modules/base-64": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", - "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==" - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -11823,14 +11818,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/charenc": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", - "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", - "engines": { - "node": "*" - } - }, "node_modules/check-more-types": { "version": "2.24.0", "resolved": "https://registry.npmjs.org/check-more-types/-/check-more-types-2.24.0.tgz", @@ -12617,14 +12604,6 @@ "node": ">= 8" } }, - "node_modules/crypt": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", - "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", - "engines": { - "node": "*" - } - }, "node_modules/crypto-browserify": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", @@ -13398,15 +13377,6 @@ "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==", "dev": true }, - "node_modules/digest-fetch": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz", - "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==", - "dependencies": { - "base-64": "^0.1.0", - "md5": "^2.3.0" - } - }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -18882,16 +18852,6 @@ "react-email": ">1.9.3" } }, - "node_modules/md5": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", - "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", - "dependencies": { - "charenc": "0.0.2", - "crypt": "0.0.2", - "is-buffer": "~1.1.6" - } - }, "node_modules/md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -19915,15 +19875,14 @@ } }, "node_modules/openai": { - "version": "4.28.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.28.0.tgz", - "integrity": "sha512-JM8fhcpmpGN0vrUwGquYIzdcEQHtFuom6sRCbbCM6CfzZXNuRk33G7KfeRAIfnaCxSpzrP5iHtwJzIm6biUZ2Q==", + "version": "4.33.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.33.0.tgz", + "integrity": "sha512-Sh4KvplkvkAREuhb8yZpohqsOo08cBBu6LNWLD8YyMxe8yCxbE+ouJYUs1X2oDPrzQGANj0rFNQYiwW9gWLBOg==", "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", - "digest-fetch": "^1.3.0", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7", diff --git a/app/package.json b/app/package.json index 9805409c7..a50ca77fa 100644 --- a/app/package.json +++ b/app/package.json @@ -73,7 +73,7 @@ "next": "14.1.3", "next-auth": "^4.24.5", "nodemailer": "^6.9.13", - "openai": "^4.28.0", + "openai": "^4.33.0", "pg": "^8.11.5", "pg-hstore": "^2.3.4", "pigeon-maps": "^0.21.3", From dae5d1948ad28d15022357cb1ab1c87bf1ff7838 Mon Sep 17 00:00:00 2001 From: Milan Gruner Date: Mon, 8 Apr 2024 15:59:47 +0200 Subject: [PATCH 16/51] feat(ui): show progress indicator for unfinished subsectors in list --- .../[lng]/[inventory]/data/[step]/page.tsx | 44 +++++++++++++------ .../[lng]/[inventory]/data/[step]/types.d.ts | 2 + 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/app/src/app/[lng]/[inventory]/data/[step]/page.tsx b/app/src/app/[lng]/[inventory]/data/[step]/page.tsx index 1cb2c1c95..4df7bc0a6 100644 --- a/app/src/app/[lng]/[inventory]/data/[step]/page.tsx +++ b/app/src/app/[lng]/[inventory]/data/[step]/page.tsx @@ -29,7 +29,9 @@ import { Button, Card, Center, + CircularProgress, Flex, + HStack, Heading, Icon, IconButton, @@ -819,18 +821,34 @@ export default function AddDataSteps({ onClick={() => onSubsectorClick(subSector)} key={subSector.subsectorId} > - - + + {subSector.completedCount > 0 && + subSector.completedCount < subSector.totalCount ? ( + + ) : ( + + )} {t(nameToI18NKey(subSector.subsectorName!))} @@ -851,7 +869,7 @@ export default function AddDataSteps({ /> } /> - + )) )} diff --git a/app/src/app/[lng]/[inventory]/data/[step]/types.d.ts b/app/src/app/[lng]/[inventory]/data/[step]/types.d.ts index 312cf570d..4267055d4 100644 --- a/app/src/app/[lng]/[inventory]/data/[step]/types.d.ts +++ b/app/src/app/[lng]/[inventory]/data/[step]/types.d.ts @@ -119,6 +119,8 @@ type DataSourceData = { type SubSectorWithRelations = SubSectorAttributes & { completed: boolean; + completedCount: number; + totalCount: number; scope: ScopeAttributes; subCategories: SubCategoryAttributes[]; }; From 599d920e49a62174492a6d6f76f9f0d6523da74b Mon Sep 17 00:00:00 2001 From: Amanda Eames Date: Tue, 9 Apr 2024 07:32:26 -0400 Subject: [PATCH 17/51] feature new custom polygons --- global-api/import_everything.sh | 3 + .../custom_polygon_subsaharan.py | 177 ++++++++++++++++++ 2 files changed, 180 insertions(+) create mode 100644 global-api/importer/custom_polygons/custom_polygon_subsaharan.py diff --git a/global-api/import_everything.sh b/global-api/import_everything.sh index 3960b1dbe..9f8f04be6 100755 --- a/global-api/import_everything.sh +++ b/global-api/import_everything.sh @@ -44,6 +44,9 @@ $python_cmd custom_polygon_importer.py \ --database_uri $DB_URI \ --zip_file_path "./Limites Ciudad-001.zip" \ --extract_to_path "./processed" + +$python_cmd custom_polygon_subsaharan.py \ + --database_uri $DB_URI \ popd # Import Carbon Monitor diff --git a/global-api/importer/custom_polygons/custom_polygon_subsaharan.py b/global-api/importer/custom_polygons/custom_polygon_subsaharan.py new file mode 100644 index 000000000..197064b60 --- /dev/null +++ b/global-api/importer/custom_polygons/custom_polygon_subsaharan.py @@ -0,0 +1,177 @@ +import zipfile +import pandas as pd +import geopandas as gpd +from shapely.geometry import LineString +from shapely import wkt +import argparse +import requests +from io import BytesIO +import io +import os +from sqlalchemy import create_engine +import osmnx as ox +from sqlalchemy.sql import text +from shapely.geometry import Polygon +from shapely.ops import unary_union + +def unzip_file(url, extract_to_path): + response = requests.get(url) + if response.status_code == 200: + with zipfile.ZipFile(BytesIO(response.content), 'r') as zip_ref: + zip_ref.extractall(extract_to_path) + print("Extraction successful.") + else: + print("Failed to download the zip file.") + +def multi_to_single(multi_polygon): + return multi_polygon.convex_hull + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--database_uri", + help="database URI (e.g. postgresql://ccglobal:@localhost/ccglobal)", + default=os.environ.get("DB_URI"), + ) + args = parser.parse_args() + + # # Part1: RW KGL + gdf_rw = ox.geocode_to_gdf('R1708283', by_osmid=True) + gdf_rw['geometry'] = gdf_rw['geometry'].apply(lambda x: x.wkt) + gdf_rw['locode'] = 'RW KGL' + df_rw = pd.DataFrame(gdf_rw) + + engine = create_engine(args.database_uri) + df_rw.to_sql('osm_staging', engine, if_exists='replace', index=False) + + # Define the UPSERT query using text() construct + upsert_query = """ + INSERT INTO osm (geometry, bbox_north, bbox_south, bbox_east, bbox_west, place_id, osm_type, osm_id, lat, lon, "class", "type", place_rank, importance, addresstype, name, display_name, locode) + SELECT geometry, bbox_north, bbox_south, bbox_east, bbox_west, place_id, osm_type, osm_id, lat, lon, "class", "type", place_rank, importance, addresstype, name, display_name, locode + FROM osm_staging + ON CONFLICT (locode) + DO UPDATE SET + geometry = EXCLUDED.geometry, + bbox_north = EXCLUDED.bbox_north, + bbox_south = EXCLUDED.bbox_south, + bbox_east = EXCLUDED.bbox_east, + bbox_west = EXCLUDED.bbox_west, + place_id = EXCLUDED.place_id, + osm_type = EXCLUDED.osm_type, + osm_id = EXCLUDED.osm_id, + lat = EXCLUDED.lat, + lon = EXCLUDED.lon, + "class" = EXCLUDED."class", + "type" = EXCLUDED."type", + place_rank = EXCLUDED.place_rank, + importance = EXCLUDED.importance, + addresstype = EXCLUDED.addresstype, + name = EXCLUDED.name, + display_name = EXCLUDED.display_name; + + DROP TABLE osm_staging + """ + + with engine.connect() as connection: + try: + result = connection.execute(text(upsert_query)) + connection.commit() + print("Query completed successfully.") + except Exception as e: + print("Error updating osm table:", e) + + # Part2: NG ABV and NG LOS + url = "https://datacatalogfiles.worldbank.org/ddh-published/0039368/DR0048906/ngaadmbndaadm2osgof20170222.geojson?versionId=2023-01-19T03:44:38.1621981Z" + gdf_ng = gpd.read_file(url) + gdf_ng = gdf_ng.to_crs("EPSG:4326") + names_to_filter = ['Abuja Municipal', 'Lagos Island'] + gdf_ng = gdf_ng[gdf_ng['admin2Name'].isin(names_to_filter)] + locode_mapping = {'Abuja Municipal': 'NG ABV', 'Lagos Island': 'NG LOS'} + gdf_ng['locode'] = gdf_ng['admin2Name'].map(locode_mapping) + gdf_ng = gdf_ng.explode() + gdf_ng = gdf_ng[['locode', 'geometry']] + gdf_ng[['bbox_west', 'bbox_south', 'bbox_east', 'bbox_north']] = gdf_ng.geometry.bounds + gdf_ng['lat'] = gdf_ng.geometry.centroid.y + gdf_ng['lon'] = gdf_ng.geometry.centroid.x + gdf_ng['area'] = gdf_ng.geometry.area + gdf_ng['type'] = 'custom' + gdf_ng['geometry'] = gdf_ng['geometry'].apply(lambda x: x.wkt) + df_ng = pd.DataFrame(gdf_ng) + + engine = create_engine(args.database_uri) + df_ng.to_sql('osm_staging', engine, if_exists='replace', index=False) + + # Define the UPSERT query using text() construct + upsert_query = """ + INSERT INTO osm (geometry, bbox_north, bbox_south, bbox_east, bbox_west, lat, lon, locode, type) + SELECT geometry, bbox_north, bbox_south, bbox_east, bbox_west, lat, lon, locode, type + FROM osm_staging + ON CONFLICT (locode) + DO UPDATE SET + geometry = EXCLUDED.geometry, + bbox_north = EXCLUDED.bbox_north, + bbox_south = EXCLUDED.bbox_south, + bbox_east = EXCLUDED.bbox_east, + bbox_west = EXCLUDED.bbox_west, + lat = EXCLUDED.lat, + lon = EXCLUDED.lon, + type = EXCLUDED.type; + + DROP TABLE osm_staging + """ + with engine.connect() as connection: + try: + result = connection.execute(text(upsert_query)) + connection.commit() + print("Query completed successfully.") + except Exception as e: + print("Error updating osm table:", e) + + # Part3: SL FNA + url = 'https://data.humdata.org/dataset/a4816317-a913-4619-b1e9-d89e21c056b4/resource/b6225ff4-80e2-428f-be5e-db5d2f43a3d6/download/sle_adm_gov_ocha_20231215_ab_shp.zip' + unzip_file(url, extract_to_path='./extracted/') + + gdf_sl = gpd.read_file('./extracted/sle_admbnda_adm3_gov_ocha_20231215.shp') + gdf_sl = gdf_sl.to_crs("EPSG:4326") + gdf_sl = gdf_sl[gdf_sl['ADM2_EN'].str.contains('Western Area Urban')] + gdf_sl = gdf_sl[~gdf_sl['ADM3_EN'].str.contains('Tasso Island')] + gdf_sl['geometry'] = gdf_sl['geometry'].apply(multi_to_single) + outer_polygon = unary_union(gdf_sl['geometry']) + outer_gdf = gpd.GeoDataFrame(geometry=[outer_polygon], crs="EPSG:4326") + outer_gdf['locode'] = 'SL FNA' + outer_gdf[['bbox_west', 'bbox_south', 'bbox_east', 'bbox_north']] = outer_gdf.geometry.bounds + outer_gdf['lat'] = outer_gdf.geometry.centroid.y + outer_gdf['lon'] = outer_gdf.geometry.centroid.x + outer_gdf['area'] = outer_gdf.geometry.area + outer_gdf['type'] = 'custom' + outer_gdf['geometry'] = outer_gdf['geometry'].apply(lambda x: x.wkt) + df_sl = pd.DataFrame(outer_gdf) + + engine = create_engine(args.database_uri) + df_sl.to_sql('osm_staging', engine, if_exists='replace', index=False) + + # Define the UPSERT query using text() construct + upsert_query = """ + INSERT INTO osm (geometry, bbox_north, bbox_south, bbox_east, bbox_west, lat, lon, locode, type) + SELECT geometry, bbox_north, bbox_south, bbox_east, bbox_west, lat, lon, locode, type + FROM osm_staging + ON CONFLICT (locode) + DO UPDATE SET + geometry = EXCLUDED.geometry, + bbox_north = EXCLUDED.bbox_north, + bbox_south = EXCLUDED.bbox_south, + bbox_east = EXCLUDED.bbox_east, + bbox_west = EXCLUDED.bbox_west, + lat = EXCLUDED.lat, + lon = EXCLUDED.lon, + type = EXCLUDED.type; + + DROP TABLE osm_staging + """ + with engine.connect() as connection: + try: + result = connection.execute(text(upsert_query)) + connection.commit() + print("Query completed successfully.") + except Exception as e: + print("Error updating osm table:", e) From 597bd5c276d7aa0570638e5abfe655f8bab8fb70 Mon Sep 17 00:00:00 2001 From: Cephas Chapa Date: Wed, 10 Apr 2024 14:07:54 +0200 Subject: [PATCH 18/51] fix(ui): remove hard coded inventory year and replace with actual value from inventory --- app/src/app/[lng]/[inventory]/data/review/page.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/src/app/[lng]/[inventory]/data/review/page.tsx b/app/src/app/[lng]/[inventory]/data/review/page.tsx index 9325f6069..1cdcaaa43 100644 --- a/app/src/app/[lng]/[inventory]/data/review/page.tsx +++ b/app/src/app/[lng]/[inventory]/data/review/page.tsx @@ -19,6 +19,7 @@ import { clear, removeSectorData } from "@/features/city/inventoryDataSlice"; import { api } from "@/services/api"; import { appendFileToFormData } from "@/util/helpers"; import { useState } from "react"; +import { configureStore } from "@reduxjs/toolkit"; export default function ReviewPage({ params: { lng }, @@ -570,7 +571,7 @@ export default function ReviewPage({ {t("review-data-label")} - 2023 {t("emissions-inventory-title")} + {inventory?.year} {t("emissions-inventory-title")} - -
-

Open Earth Foundation is a nonprofit public benefit corporation from California, USA. EIN: 85-3261449

- - - - -`; - - if (process.env.NODE_ENV !== "test") { - await service.sendEmail({ - to: process.env.ADMIN_EMAILS!, - subject: "CityCatalyst File Upload", - text: "City Catalyst", - html: emailTemplate, + const newFileData = { + id: userFile.id, + userId: userFile.userId!, + cityId: userFile.cityId!, + fileReference: userFile.fileReference!, + url: userFile.url!, + sector: userFile.sector!, + subsectors: userFile.subsectors!, + scopes: userFile.scopes!, + fileName: userFile.fileName!, + lastUpdated: userFile.lastUpdated!, + status: userFile.status!, + gpcRefNo: userFile.gpcRefNo!, + file: { + fileName: file.name, + size: file.size, + fileType: userFile.fileType!, + }, + }; + const host = process.env.HOST ?? "http://localhost:3000"; + + const emailTemplate = ` + + + + + + Email Notification + + +
+ + + +

CityCatalyst

+

${user?.name} From ${city.name} Uploaded New Files For Review

+

Hi ${process.env.ADMIN_NAMES},

+

${user?.name} (${user?.email}) has uploaded files in CityCatalyst for revision and to upload to their inventories.

+ +
+
${file.name}

${bytesToMB(file.size)}
${newFileData.subsectors.map((item: string) => `${item}`)}
+
+ + + +
+

Open Earth Foundation is a nonprofit public benefit corporation from California, USA. EIN: 85-3261449

+
+ + + `; + + if (process.env.NODE_ENV !== "test") { + await service.sendEmail({ + to: process.env.ADMIN_EMAILS!, + subject: "CityCatalyst File Upload", + text: "City Catalyst", + html: emailTemplate, + }); + } + + return NextResponse.json({ + data: newFileData, }); - } - - return NextResponse.json({ - data: newFileData, - }); -}); + }, +); From 1744d5b384244c6c603bbcd136750e5a6c08e336 Mon Sep 17 00:00:00 2001 From: Milan Gruner Date: Mon, 15 Apr 2024 12:42:56 +0200 Subject: [PATCH 38/51] fix(test): add user to city in UserFile tests --- app/tests/api/userfile.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app/tests/api/userfile.test.ts b/app/tests/api/userfile.test.ts index 485f0b105..fab3892f1 100644 --- a/app/tests/api/userfile.test.ts +++ b/app/tests/api/userfile.test.ts @@ -65,8 +65,16 @@ describe("UserFile API", () => { setupTests(); await db.initialize(); await db.models.UserFile.destroy({ where: { userId: testUserID } }); - await db.models.User.upsert({ userId: testUserID, name: "TEST_USER" }); - await db.models.City.upsert({ cityId: testCityID, name: "TEST_CITY" }); + const [user] = await db.models.User.upsert({ + userId: testUserID, + name: "TEST_USER", + }); + const [city] = await db.models.City.upsert({ + cityId: testCityID, + name: "TEST_CITY", + }); + await user.addCity(city); + const service = new NotificationService(); service.sendEmail = async () => { return { success: true, messageId: "send" }; From 6a73d04d9fb7cd5c7d297bc922411446d925bb5f Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 15 Apr 2024 08:22:53 -0400 Subject: [PATCH 39/51] feat: add area to city boundaries and own endpoint Signed-off-by: Evan Prodromou --- global-api/requirements.txt | 3 +- global-api/routes/city_boundaries_endpoint.py | 51 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/global-api/requirements.txt b/global-api/requirements.txt index 1fbc7ede4..a5e16c878 100644 --- a/global-api/requirements.txt +++ b/global-api/requirements.txt @@ -26,4 +26,5 @@ dns-cache==0.3.* openpyxl==3.1.* pytest==8.1.1 pytest-cov==4.1.0 -httpx==0.27.0 \ No newline at end of file +httpx==0.27.0 +pyproj==3.6.* \ No newline at end of file diff --git a/global-api/routes/city_boundaries_endpoint.py b/global-api/routes/city_boundaries_endpoint.py index 750714cc6..ce999e4d2 100644 --- a/global-api/routes/city_boundaries_endpoint.py +++ b/global-api/routes/city_boundaries_endpoint.py @@ -7,6 +7,10 @@ from decimal import Decimal from shapely.geometry import Point, shape from shapely.wkt import loads +import pyproj +from functools import partial +from shapely.ops import transform +import math api_router = APIRouter(prefix="/api/v0") @@ -21,6 +25,37 @@ def db_query(locode): return row +def epsg_code(polygon): + """Calculate the UTM zone and corresponding EPSG code for a polygon""" + # Calculate centroid of the polygon + centroid = polygon.centroid + longitude = centroid.x + latitude = centroid.y + + # Calculate UTM zone from centroid longitude + zone_number = math.floor((longitude + 180) / 6) + 1 + + # Determine the hemisphere and corresponding EPSG code + if latitude >= 0: + code = 32600 + zone_number + else: + code = 32700 + zone_number + + return code + +def get_area(geometry): + polygon = loads(geometry) + code = epsg_code(polygon) + + proj = partial( + pyproj.transform, + pyproj.Proj("epsg:4326"), # lat/lon + pyproj.Proj(f'epsg:{code}') + ) + + utm_polygon = transform(proj, polygon) + area = utm_polygon.area / 10.0**6 # in km^2 + return area @api_router.get("/cityboundary/city/{locode}") def get_city_boundary(locode: str): @@ -29,12 +64,28 @@ def get_city_boundary(locode: str): if not city: raise HTTPException(status_code=404, detail="City boundary not found") + area = get_area(city.geometry) + return { "city_geometry": city.geometry, "bbox_north": city.bbox_north, "bbox_south": city.bbox_south, "bbox_east": city.bbox_east, "bbox_west": city.bbox_west, + "area": area + } + +@api_router.get("/cityboundary/city/{locode}/area") +def get_city_area(locode: str): + city = db_query(locode) + + if not city: + raise HTTPException(status_code=404, detail="City boundary not found") + + area = get_area(city.geometry) + + return { + "area": area } @api_router.get("/cityboundary/locode/{lat}/{lon}") From 8a2ea1f35aa2ad016589a37d5418cddfb8065fb6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 12:59:58 +0000 Subject: [PATCH 40/51] chore(deps): bump pytest-cov from 4.1.0 to 5.0.0 in /global-api Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 4.1.0 to 5.0.0. - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v4.1.0...v5.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- global-api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global-api/requirements.txt b/global-api/requirements.txt index a5e16c878..e3c7e972a 100644 --- a/global-api/requirements.txt +++ b/global-api/requirements.txt @@ -25,6 +25,6 @@ xlrd==2.0.* dns-cache==0.3.* openpyxl==3.1.* pytest==8.1.1 -pytest-cov==4.1.0 +pytest-cov==5.0.0 httpx==0.27.0 pyproj==3.6.* \ No newline at end of file From 27423d3092a5dee6ff9861deaa86d761736c6ec2 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 15 Apr 2024 09:22:50 -0400 Subject: [PATCH 41/51] feat: function for estimating value from a series Signed-off-by: Evan Prodromou --- app/src/util/series.ts | 46 ++++++++++++++++++++++++++++++ app/tests/series.test.ts | 60 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 app/src/util/series.ts create mode 100644 app/tests/series.test.ts diff --git a/app/src/util/series.ts b/app/src/util/series.ts new file mode 100644 index 000000000..4f53ea722 --- /dev/null +++ b/app/src/util/series.ts @@ -0,0 +1,46 @@ +export function estimate(series: { year: number, value: number }[], year: number): number | null { + + // if it's in the series, just return the value + + const exact = series.find(e => e.year === year); + if (exact) { + return exact.value; + } + + // we need at least two points to interpolate or extrapolate + + if (series.length < 2) { + return null; + } + + if (year < series[0].year) { // extrapolate backwards + let first = series[0] + let next = series.find(e => e.year - first.year >= first.year - year) + if (!next) { + return null + } + let rate = (1.0 * (next.value - first.value)) / (next.year - first.year); + let value = first.value - rate * (first.year - year) + return Math.round(value) + } else if (year > series[series.length - 1].year) { // extrapolate forwards + let last = series[series.length - 1] + let prev = series.findLast(e => last.year - e.year >= year - last.year) + if (!prev) { + return null + } + let rate = (1.0 * (last.value - prev.value)) / (last.year - prev.year); + let value = last.value + rate * (year - last.year) + return Math.round(value) + } else { // interpolate + let prev = series.findLast(e => e.year < year) + let next = series.find(e => e.year > year) + if (!prev || !next) { + return null + } + let rate = (1.0 * (next.value - prev.value)) / (next.year - prev.year); + let value = prev.value + rate * (year - prev.year) + return Math.round(value) + } + + return null; +} \ No newline at end of file diff --git a/app/tests/series.test.ts b/app/tests/series.test.ts new file mode 100644 index 000000000..76da43a60 --- /dev/null +++ b/app/tests/series.test.ts @@ -0,0 +1,60 @@ +import { describe, it } from "node:test" +import assert from "node:assert/strict" +import { estimate } from "@/util/series" + +describe("Series", () => { + let series0 = [ + { year: 2000, value: 100 } + ]; + + let series = [ + { year: 2000, value: 100 }, + { year: 2010, value: 200 }, + { year: 2020, value: 400 } + ]; + + it("should return exact value for small series on match", () => { + let result = estimate(series0, 2000); + assert.strictEqual(result, 100); + }) + + it("should return null for small series on no match", () => { + let result = estimate(series0, 2001); + assert.strictEqual(result, null); + }) + + it("should return exact value", () => { + let result = estimate(series, 2010); + assert.strictEqual(result, 200); + }) + + it("should return interpolated value at low rate", () => { + let result = estimate(series, 2005); + assert.strictEqual(result, 150); + }) + + it("should return interpolated value at high rate", () => { + let result = estimate(series, 2015); + assert.strictEqual(result, 300); + }) + + it("should return extrapolated value at low rate", () => { + let result = estimate(series, 1995); + assert.strictEqual(result, 50); + }) + + it("should return extrapolated value at high rate", () => { + let result = estimate(series, 2025); + assert.strictEqual(result, 500); + }) + + it("should return null if below safe extrapolation range", () => { + let result = estimate(series, 1979); + assert.strictEqual(result, null); + }) + + it("should return null if above safe extrapolation range", () => { + let result = estimate(series, 2041); + assert.strictEqual(result, null); + }) +}) \ No newline at end of file From dc9072662b995ab64e9bce1f6d765fc8f28e8dad Mon Sep 17 00:00:00 2001 From: Cephas Chapa Date: Mon, 15 Apr 2024 15:27:07 +0200 Subject: [PATCH 42/51] fix(ui): show right flag for city --- .../app/[lng]/[inventory]/settings/page.tsx | 1 + app/src/components/Tabs/my-files-tab.tsx | 54 ++++++------------- 2 files changed, 16 insertions(+), 39 deletions(-) diff --git a/app/src/app/[lng]/[inventory]/settings/page.tsx b/app/src/app/[lng]/[inventory]/settings/page.tsx index 6e24cd94e..2d0543184 100644 --- a/app/src/app/[lng]/[inventory]/settings/page.tsx +++ b/app/src/app/[lng]/[inventory]/settings/page.tsx @@ -149,6 +149,7 @@ export default function Settings({ t={t} userInfo={userInfo!} userFiles={userFiles!} + inventory={inventory!} /> = ({ @@ -73,22 +75,9 @@ const MyFilesTab: FC = ({ lng, userInfo, userFiles, + inventory }) => { - const [cities, setCities] = useState>([]); - - useEffect(() => { - const data = [ - { - id: "1", - name: "CITY", - state: "Test Region", - country: "TEST COUNTRY", - lastUpdated: "2023-10-10T12:05:41.340Z", - }, - ]; - setCities(data); - }, []); - + const getYears = userFiles?.map((item: any): number => { const date = new Date(item.lastUpdated); return date.getFullYear(); @@ -119,14 +108,6 @@ const MyFilesTab: FC = ({ onClose: onFileDeleteModalClose, } = useDisclosure(); - const [cityData, setCityData] = useState({ - id: "", - name: "", - state: "", - country: "", - lastUpdated: "", - }); - const [fileData, setFileData] = useState(); return ( @@ -175,12 +156,7 @@ const MyFilesTab: FC = ({ gap="36px" > - {cities.map(({ name, id, country, lastUpdated, state }) => ( - setCityData({ name, country, id, lastUpdated, state }) - } - key={id} sx={{ w: "223px", justifyContent: "left", @@ -204,15 +180,12 @@ const MyFilesTab: FC = ({ borderColor: "content.link", }} > - {name} + {inventory?.city.name} - ))} - {cities.map((city) => ( = ({ > - + = ({ fontFamily="heading" fontStyle="normal" > - {cityData.name} + {inventory?.city.name} From 9313f32dccf2997ee00902bef8a478df984a1846 Mon Sep 17 00:00:00 2001 From: Cephas Chapa Date: Mon, 15 Apr 2024 15:45:42 +0200 Subject: [PATCH 43/51] fix(ui): remove extra brackets causing build errors --- app/src/components/Tabs/my-files-tab.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/src/components/Tabs/my-files-tab.tsx b/app/src/components/Tabs/my-files-tab.tsx index 1131bea43..1dc46c7b3 100644 --- a/app/src/components/Tabs/my-files-tab.tsx +++ b/app/src/components/Tabs/my-files-tab.tsx @@ -326,7 +326,7 @@ const MyFilesTab: FC = ({ fontSize="body.md" > {filteredData.map((file: any) => ( - + = ({ )} - ))} From 33fbe016d8d907b5cac9486ad183fc03771fd03d Mon Sep 17 00:00:00 2001 From: Cephas Chapa Date: Mon, 15 Apr 2024 16:04:43 +0200 Subject: [PATCH 44/51] feat: add area in return response --- app/src/app/api/v0/city/[city]/boundary/route.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/app/api/v0/city/[city]/boundary/route.ts b/app/src/app/api/v0/city/[city]/boundary/route.ts index d045c92ae..73f076df9 100644 --- a/app/src/app/api/v0/city/[city]/boundary/route.ts +++ b/app/src/app/api/v0/city/[city]/boundary/route.ts @@ -29,7 +29,7 @@ export const GET = apiHandler(async (_req, { params }) => { data.bbox_north, ]; - return NextResponse.json({ data: geoJson, boundingBox }); + return NextResponse.json({ data: geoJson, boundingBox, area: data.area }); } catch (error: any) { logger.error(error); return NextResponse.json({ error: error.message }); From cbad07f39095bbd32f2f713254f64cf3ee33e904 Mon Sep 17 00:00:00 2001 From: Cephas Chapa Date: Mon, 15 Apr 2024 16:24:33 +0200 Subject: [PATCH 45/51] fix(ui): show area in ui --- app/src/app/[lng]/[inventory]/page.tsx | 41 +++++++++++++++----------- app/src/services/api.ts | 3 +- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/app/src/app/[lng]/[inventory]/page.tsx b/app/src/app/[lng]/[inventory]/page.tsx index 30067c51e..61111d923 100644 --- a/app/src/app/[lng]/[inventory]/page.tsx +++ b/app/src/app/[lng]/[inventory]/page.tsx @@ -119,6 +119,10 @@ export default function Home({ params: { lng } }: { params: { lng: string } }) { { skip: !inventory?.cityId || !inventory?.year }, ); + const { data: cityArea, isLoading: isCityAreaLoading } = api.useGetCityBoundaryQuery(inventory?.city.locode!, { + skip: !inventory?.city.locode, + }); + let totalProgress = 0, thirdPartyProgress = 0, uploadedProgress = 0; @@ -384,27 +388,28 @@ export default function Home({ params: { lng } }: { params: { lng: string } }) { /> - {!city?.area ? ( + {cityArea?.area ? ( - N/A - + fontFamily="heading" + color="base.light" + fontSize="headline.sm" + fontWeight="semibold" + lineHeight="32" + > + {Math.round(cityArea?.area)} + km2 + ) : ( + - {city?.area} - km2 - + fontFamily="heading" + color="border.neutral" + fontSize="headline.sm" + fontWeight="semibold" + lineHeight="32" + > + N/A + )} diff --git a/app/src/services/api.ts b/app/src/services/api.ts index 253e6af11..170b6530a 100644 --- a/app/src/services/api.ts +++ b/app/src/services/api.ts @@ -42,13 +42,14 @@ export const api = createApi({ transformResponse: (response: { data: CityAttributes }) => response.data, }), getCityBoundary: builder.query< - { data: GeoJSON; boundingBox: BoundingBox }, + { data: GeoJSON; boundingBox: BoundingBox; area: number }, string >({ query: (cityId) => `city/${cityId}/boundary`, transformResponse: (response: { data: GeoJSON; boundingBox: BoundingBox; + area: number }) => response, }), getInventory: builder.query({ From f84c79ffaa0f35106043dcbf80b73d1d642a9324 Mon Sep 17 00:00:00 2001 From: Cephas Chapa Date: Mon, 15 Apr 2024 16:56:43 +0200 Subject: [PATCH 46/51] fix: show area in onboarding confirm page --- app/src/app/[lng]/onboarding/setup/page.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/app/src/app/[lng]/onboarding/setup/page.tsx b/app/src/app/[lng]/onboarding/setup/page.tsx index 542d80fff..e830bacb7 100644 --- a/app/src/app/[lng]/onboarding/setup/page.tsx +++ b/app/src/app/[lng]/onboarding/setup/page.tsx @@ -7,6 +7,7 @@ import { useTranslation } from "@/i18n/client"; import { useAppDispatch, useAppSelector } from "@/lib/hooks"; import type { CityAttributes } from "@/models/City"; import { + api, useAddCityMutation, useAddCityPopulationMutation, useAddInventoryMutation, @@ -641,7 +642,7 @@ function ConfirmStep({ {area > 0 ? ( <> {" "} - {area}km2 + {Math.round(area)}km2 ) : ( "N/A" @@ -716,6 +717,10 @@ export default function OnboardingSetup({ const regionPopulationYear = watch("regionPopulationYear"); const countryPopulationYear = watch("countryPopulationYear"); + const { data: cityArea, isLoading: isCityAreaLoading } = api.useGetCityBoundaryQuery(data.locode!, { + skip: !data.locode, + }); + const onConfirm = async () => { // save data in backend setConfirming(true); @@ -831,7 +836,7 @@ export default function OnboardingSetup({ cityName={getValues("city")} t={t} locode={data.locode} - area={ocCityData?.area!} + area={cityArea?.area!} population={cityPopulation} /> )} From 7ac13d2df2c70dd71487a4694c501ad7a3ca2d58 Mon Sep 17 00:00:00 2001 From: Cephas Chapa Date: Tue, 16 Apr 2024 10:44:39 +0200 Subject: [PATCH 47/51] fix: add area to city during onboarding --- app/src/app/[lng]/[inventory]/page.tsx | 43 +++++++++------------ app/src/app/[lng]/onboarding/setup/page.tsx | 2 +- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/app/src/app/[lng]/[inventory]/page.tsx b/app/src/app/[lng]/[inventory]/page.tsx index 61111d923..3fac76b1e 100644 --- a/app/src/app/[lng]/[inventory]/page.tsx +++ b/app/src/app/[lng]/[inventory]/page.tsx @@ -119,10 +119,6 @@ export default function Home({ params: { lng } }: { params: { lng: string } }) { { skip: !inventory?.cityId || !inventory?.year }, ); - const { data: cityArea, isLoading: isCityAreaLoading } = api.useGetCityBoundaryQuery(inventory?.city.locode!, { - skip: !inventory?.city.locode, - }); - let totalProgress = 0, thirdPartyProgress = 0, uploadedProgress = 0; @@ -388,28 +384,27 @@ export default function Home({ params: { lng } }: { params: { lng: string } }) { /> - {cityArea?.area ? ( - - {Math.round(cityArea?.area)} - km2 - + {inventory?.city.area! == 0 || inventory?.city.area === null ? ( + + N/A + ) : ( - - N/A - + fontFamily="heading" + color="base.light" + fontSize="headline.sm" + fontWeight="semibold" + lineHeight="32" + > + {Math.round(inventory?.city.area!)} + km2 + )} diff --git a/app/src/app/[lng]/onboarding/setup/page.tsx b/app/src/app/[lng]/onboarding/setup/page.tsx index e830bacb7..6938c861d 100644 --- a/app/src/app/[lng]/onboarding/setup/page.tsx +++ b/app/src/app/[lng]/onboarding/setup/page.tsx @@ -739,7 +739,7 @@ export default function OnboardingSetup({ city = await addCity({ name: data.name, locode: data.locode!, - area, + area: Math.round(cityArea?.area!), region, country, }).unwrap(); From f8d7695bf7ce5d1d51f8adac78f6d6854009aa75 Mon Sep 17 00:00:00 2001 From: Cephas Chapa Date: Tue, 16 Apr 2024 10:48:44 +0200 Subject: [PATCH 48/51] fix: added conditional checks for area during onboarding --- app/src/app/[lng]/onboarding/setup/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/app/[lng]/onboarding/setup/page.tsx b/app/src/app/[lng]/onboarding/setup/page.tsx index 6938c861d..cb7762006 100644 --- a/app/src/app/[lng]/onboarding/setup/page.tsx +++ b/app/src/app/[lng]/onboarding/setup/page.tsx @@ -639,7 +639,7 @@ function ConfirmStep({ - {area > 0 ? ( + {area && area > 0 ? ( <> {" "} {Math.round(area)}km2 From 408e7e7a6ed33f9f52b6c85a0a17dad528dcccef Mon Sep 17 00:00:00 2001 From: Cephas Chapa Date: Tue, 16 Apr 2024 12:39:24 +0200 Subject: [PATCH 49/51] fix: manaul input form state errors --- .../data/[step]/SubsectorDrawer.tsx | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/app/src/app/[lng]/[inventory]/data/[step]/SubsectorDrawer.tsx b/app/src/app/[lng]/[inventory]/data/[step]/SubsectorDrawer.tsx index 27851c81f..e38ba91ec 100644 --- a/app/src/app/[lng]/[inventory]/data/[step]/SubsectorDrawer.tsx +++ b/app/src/app/[lng]/[inventory]/data/[step]/SubsectorDrawer.tsx @@ -44,7 +44,7 @@ import type { FetchBaseQueryError } from "@reduxjs/toolkit/query"; import type { TFunction } from "i18next"; import type { RefObject } from "react"; import React, { useEffect } from "react"; -import { SubmitHandler, useForm } from "react-hook-form"; +import { FormProvider, SubmitHandler, useForm } from "react-hook-form"; import { EmissionsForm } from "./EmissionsForm"; import type { ActivityData, @@ -242,13 +242,15 @@ export function SubsectorDrawer({ const { register, handleSubmit, - formState: { errors, isSubmitting, isDirty }, + formState: { errors, isSubmitting, isDirty, isValid }, watch, reset, control, setValue, } = useForm(); + const methods = useForm() + const scopeData = watch("subcategoryData"); const onTryClose = () => { @@ -506,25 +508,28 @@ export function SubsectorDrawer({ {scopes?.map((scope) => ( +

- {isScopeCompleted(scope.value, scopeData) ? ( - - - - ) : ( - - - - )} + { + !isDirty && !isScopeCompleted(scope.value, scopeData) ? "": isScopeCompleted(scope.value, scopeData) && isValid && !isDirty? ( + + + + ) : ( + + + + ) + } {scope.label} - {isScopeCompleted(scope.value, scopeData) ? ( - "" - ) : ( + {!isDirty && isScopeCompleted(scope.value, scopeData) ? null : isScopeCompleted(scope.value, scopeData) && isValid && !isDirty? ( {t("save-missing-scope-info")} - )} + ): ""} + ))} From fd3c64d726c65df4a612bb452d9e863fa21cbfbe Mon Sep 17 00:00:00 2001 From: Cephas Chapa Date: Tue, 16 Apr 2024 12:51:03 +0200 Subject: [PATCH 50/51] fix: removed console, reverted formprovider component --- .../app/[lng]/[inventory]/data/[step]/SubsectorDrawer.tsx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/src/app/[lng]/[inventory]/data/[step]/SubsectorDrawer.tsx b/app/src/app/[lng]/[inventory]/data/[step]/SubsectorDrawer.tsx index e38ba91ec..26a068c73 100644 --- a/app/src/app/[lng]/[inventory]/data/[step]/SubsectorDrawer.tsx +++ b/app/src/app/[lng]/[inventory]/data/[step]/SubsectorDrawer.tsx @@ -44,7 +44,7 @@ import type { FetchBaseQueryError } from "@reduxjs/toolkit/query"; import type { TFunction } from "i18next"; import type { RefObject } from "react"; import React, { useEffect } from "react"; -import { FormProvider, SubmitHandler, useForm } from "react-hook-form"; +import { SubmitHandler, useForm } from "react-hook-form"; import { EmissionsForm } from "./EmissionsForm"; import type { ActivityData, @@ -249,8 +249,6 @@ export function SubsectorDrawer({ setValue, } = useForm(); - const methods = useForm() - const scopeData = watch("subcategoryData"); const onTryClose = () => { @@ -508,7 +506,6 @@ export function SubsectorDrawer({ {scopes?.map((scope) => ( -

@@ -611,7 +608,6 @@ export function SubsectorDrawer({ - ))} From 5e1619e3f13e71f4b112d29eed9063ad031c6e59 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 16 Apr 2024 09:15:29 -0400 Subject: [PATCH 51/51] fix: remove unnecessary casts, rounding from series.estimate Signed-off-by: Evan Prodromou --- app/src/util/series.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/src/util/series.ts b/app/src/util/series.ts index 4f53ea722..02453bc98 100644 --- a/app/src/util/series.ts +++ b/app/src/util/series.ts @@ -19,27 +19,27 @@ export function estimate(series: { year: number, value: number }[], year: number if (!next) { return null } - let rate = (1.0 * (next.value - first.value)) / (next.year - first.year); + let rate = (next.value - first.value) / (next.year - first.year); let value = first.value - rate * (first.year - year) - return Math.round(value) + return value } else if (year > series[series.length - 1].year) { // extrapolate forwards let last = series[series.length - 1] let prev = series.findLast(e => last.year - e.year >= year - last.year) if (!prev) { return null } - let rate = (1.0 * (last.value - prev.value)) / (last.year - prev.year); + let rate = (last.value - prev.value) / (last.year - prev.year); let value = last.value + rate * (year - last.year) - return Math.round(value) + return value } else { // interpolate let prev = series.findLast(e => e.year < year) let next = series.find(e => e.year > year) if (!prev || !next) { return null } - let rate = (1.0 * (next.value - prev.value)) / (next.year - prev.year); + let rate = (next.value - prev.value) / (next.year - prev.year); let value = prev.value + rate * (year - prev.year) - return Math.round(value) + return value } return null;