From 39d60fc5587e80aa6b49065d2b4c0fdbe0b513e6 Mon Sep 17 00:00:00 2001 From: Dan Scales Date: Mon, 26 Aug 2024 13:04:16 -0700 Subject: [PATCH 1/3] Disable export to GEE of default integrated_alerts COG Hotfix to disable export for now, because the meta-data of the integrated alerts COG is too large (> 10Mbyte), since GEE recently lowered the max size of meta-data. --- src/datapump/sync/sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/datapump/sync/sync.py b/src/datapump/sync/sync.py index c55d202..f2e87e2 100644 --- a/src/datapump/sync/sync.py +++ b/src/datapump/sync/sync.py @@ -252,7 +252,8 @@ def build_jobs(self, config: DatapumpConfig) -> List[Job]: resampling="mode", implementation="default", blocksize=1024, - export_to_gee=True, + # Disable export to GEE until COG metadata is reduced to < 10Mbytes + export_to_gee=False, ), # Created from the "intensity" asset CogAssetParameters( From 548048da83bc41812d2b57e738abd3b994b6e236 Mon Sep 17 00:00:00 2001 From: Daniel Mannarino Date: Fri, 16 Aug 2024 22:07:28 -0400 Subject: [PATCH 2/3] docker-compose -> docker compose --- .github/workflows/terraform_destroy.yaml | 2 +- scripts/infra | 4 ++-- scripts/test | 16 ++++++++-------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/terraform_destroy.yaml b/.github/workflows/terraform_destroy.yaml index 0b3aea3..bed1477 100644 --- a/.github/workflows/terraform_destroy.yaml +++ b/.github/workflows/terraform_destroy.yaml @@ -8,4 +8,4 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Destroy state and delete workspace - run: docker-compose -f docker/terraform/docker-compose.yml run --entrypoint delete_workspace --rm terraform github.ref + run: docker compose -f docker/terraform/docker-compose.yml run --entrypoint delete_workspace --rm terraform github.ref diff --git a/scripts/infra b/scripts/infra index c7ae9a3..a669a41 100755 --- a/scripts/infra +++ b/scripts/infra @@ -2,5 +2,5 @@ set -e -docker-compose -f terraform/docker/docker-compose.yml build -docker-compose -f terraform/docker/docker-compose.yml run --rm terraform "$@" +docker compose -f terraform/docker/docker-compose.yml build +docker compose -f terraform/docker/docker-compose.yml run --rm terraform "$@" diff --git a/scripts/test b/scripts/test index 871829c..063029c 100755 --- a/scripts/test +++ b/scripts/test @@ -27,21 +27,21 @@ if [ "${BUILD}" = true ]; then # pre-download bigdata image, otherwise localstack try to download it when you make your first EMR request, # and cause boto3 to timeout because it's a ~2.5 GB image docker pull localstack/bigdata - docker-compose -f docker-compose-test.yml --project-name gfw-datapump_test build --no-cache datapump_test + docker compose -f docker-compose-test.yml --project-name gfw-datapump_test build --no-cache datapump_test fi -#docker-compose -f docker-compose-test.yml --project-name gfw-datapump_test restart mock_server +#docker compose -f docker-compose-test.yml --project-name gfw-datapump_test restart mock_server if [ "${DUMP_TO_STDOUT}" = true ]; then - docker-compose -f docker-compose-test.yml --project-name gfw-datapump_test run datapump_test || failed=yes - docker-compose -f docker-compose-test.yml --project-name gfw-datapump_test logs -t + docker compose -f docker-compose-test.yml --project-name gfw-datapump_test run datapump_test || failed=yes + docker compose -f docker-compose-test.yml --project-name gfw-datapump_test logs -t else - docker-compose -f docker-compose-test.yml --project-name gfw-datapump_test run datapump_test | tee tests/logs/tests.log || failed=yes - docker-compose -f docker-compose-test.yml --project-name gfw-datapump_test logs --no-color localstack > tests/logs/localstack.log - docker-compose -f docker-compose-test.yml --project-name gfw-datapump_test logs --no-color mock_server > tests/logs/mock_server.log + docker compose -f docker-compose-test.yml --project-name gfw-datapump_test run datapump_test | tee tests/logs/tests.log || failed=yes + docker compose -f docker-compose-test.yml --project-name gfw-datapump_test logs --no-color localstack > tests/logs/localstack.log + docker compose -f docker-compose-test.yml --project-name gfw-datapump_test logs --no-color mock_server > tests/logs/mock_server.log fi -docker-compose -f docker-compose-test.yml --project-name gfw-datapump_test down +docker compose -f docker-compose-test.yml --project-name gfw-datapump_test down rm tests/terraform/terraform.tfstate* [[ -z "${failed:-}" ]] || exit 1 \ No newline at end of file From 62ab1757eb9fb32cb0099b4bef1c8e980a48c961 Mon Sep 17 00:00:00 2001 From: Dan Scales Date: Wed, 28 Aug 2024 15:16:03 -0700 Subject: [PATCH 3/3] Make Dockerfile change. --- src/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dockerfile b/src/Dockerfile index 5319d55..c332b78 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -15,7 +15,7 @@ RUN pip install . -t python # to change the hash of the file and get TF to realize it needs to be # redeployed. Ticket for a better solution: # https://gfw.atlassian.net/browse/GTC-1250 -# change 7 +# change 8 RUN yum install -y zip geos-devel