From a43501f4b6906c2812cf62a62770772099605f53 Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Wed, 19 Jan 2022 11:01:37 +0100 Subject: [PATCH 01/15] Add missing newline --- maintenance/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintenance/requirements.txt b/maintenance/requirements.txt index ffa77af7b..739dc3537 100644 --- a/maintenance/requirements.txt +++ b/maintenance/requirements.txt @@ -1,2 +1,2 @@ psycopg2==2.8.3 -requests==2.24.0 \ No newline at end of file +requests==2.24.0 From 61853ea069cf80cd5080dc355f7cfc87f29a3ecc Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Wed, 19 Jan 2022 11:01:57 +0100 Subject: [PATCH 02/15] Unbreak Python 3.8 see https://github.com/psycopg/psycopg2/releases/tag/2_8_4 --- maintenance/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maintenance/requirements.txt b/maintenance/requirements.txt index 739dc3537..7a98cb02a 100644 --- a/maintenance/requirements.txt +++ b/maintenance/requirements.txt @@ -1,2 +1,2 @@ -psycopg2==2.8.3 +psycopg2==2.8.4 requests==2.24.0 From 01b76967ba94d73db1d6aeea6dc70c47753e413f Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Wed, 19 Jan 2022 18:09:22 +0100 Subject: [PATCH 03/15] fix incorrectly gitignored file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 56060de36..c523cb808 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ etl/images/* etl/*.geojson etl/**/*.csv etl/**/*.txt +!etl/**/requirements.txt etl/**/*.xls etl/**/*.xlsx etl/**/*.zip From 306f030e86a9e055626d5b91fb208aa843038506 Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Wed, 19 Jan 2022 18:10:04 +0100 Subject: [PATCH 04/15] bump one another psycopq2 look reasoably safe, but untested --- etl/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl/requirements.txt b/etl/requirements.txt index d1e037461..797c78e95 100644 --- a/etl/requirements.txt +++ b/etl/requirements.txt @@ -1,7 +1,7 @@ # Python packages for etl fiona==1.7.13 osmnx==0.8.1 -psycopg2==2.7.5 +psycopg2==2.8.4 # test is it working, used to be 2.7.5 (no obviously borken things) shapely==1.6.4 retrying==1.3.3 requests==2.23.0 From 699be734297b39341d48aeee9fdc10f82a173afa Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Wed, 19 Jan 2022 18:11:12 +0100 Subject: [PATCH 05/15] update Vagrant box to Ubuntu 20.04 --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 181e3aec6..1a0074863 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -12,7 +12,7 @@ Vagrant.configure("2") do |config| # Every Vagrant development environment requires a box. You can search for # boxes at https://vagrantcloud.com/search. - config.vm.box = "bento/ubuntu-18.04" + config.vm.box = "ubuntu/focal64" # Create a forwarded port mapping which allows access to a specific port # within the machine from a port on the host machine and only allow access From aea961d348aad693f48c53d2df547ead955feb44 Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Wed, 19 Jan 2022 18:11:47 +0100 Subject: [PATCH 06/15] Edit correct config files --- provision/vm_provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/provision/vm_provision.sh b/provision/vm_provision.sh index 88f08d70a..50db062ee 100644 --- a/provision/vm_provision.sh +++ b/provision/vm_provision.sh @@ -73,9 +73,9 @@ service postgresql start # Ensure en_US locale exists locale-gen en_US.UTF-8 # Database config to listen on network connection -sed -i "s/#\?listen_address.*/listen_addresses '*'/" /etc/postgresql/10/main/postgresql.conf +sed -i "s/#\?listen_address.*/listen_addresses '*'/" /etc/postgresql/12/main/postgresql.conf # Allow password connections from any IP (so includes host) -echo "host all all all md5" >> /etc/postgresql/10/main/pg_hba.conf +echo "host all all all md5" >> /etc/postgresql/12/main/pg_hba.conf # Restart postgres to pick up config changes service postgresql restart From de641c64857e718b567145953d552087454b17f7 Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Wed, 19 Jan 2022 20:38:02 +0100 Subject: [PATCH 07/15] Workaround for nasty vagrant-npm interaction --- provision/vm_provision.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/provision/vm_provision.sh b/provision/vm_provision.sh index 50db062ee..93c7f1cec 100644 --- a/provision/vm_provision.sh +++ b/provision/vm_provision.sh @@ -120,8 +120,20 @@ chown -R vagrant:vagrant /home/vagrant/colouringlondon # Install latest release of npm npm install -g npm@next +# https://stackoverflow.com/questions/69094604/npm-fails-during-npm-install-with-npm-err-maximum-call-stack-size-exceeded +# npm install on files from host at least sometimes goes down with +# +# 957 verbose stack RangeError: Maximum call stack size exceeded +# 957 verbose stack at RegExp.test () +# 957 verbose stack at isDepOptional (/usr/local/lib/node/node-v12.18.1/lib/node_modules/npm/lib/install/deps.js:432:45) +# 957 verbose stack at failedDependency (/usr/local/lib/node/node-v12.18.1/lib/node_modules/npm/lib/install/deps.js:441:9) +# (..) +# +# which is avoided when remote files are first copied within VM +cp /vagrant/app /home/vagrant/ -r + # Local fixed install of node modules -cd /vagrant/app && npm install +cd /home/vagrant/app && npm install # From 42e5da46e6f83ba57858c3113f8634ebfb5b69fc Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Wed, 19 Jan 2022 20:43:06 +0100 Subject: [PATCH 08/15] npm@next is gone ideally we would upgrade to some later npm version, but lets do one thing at once --- provision/vm_provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/provision/vm_provision.sh b/provision/vm_provision.sh index 93c7f1cec..cf1e48790 100644 --- a/provision/vm_provision.sh +++ b/provision/vm_provision.sh @@ -117,8 +117,8 @@ chown -R vagrant:vagrant /home/vagrant/colouringlondon # Install node modules # -# Install latest release of npm -npm install -g npm@next +# Install release v6 of npm +npm install -g npm@6 # TODO: upgrade to later npm to match intended "latest" # https://stackoverflow.com/questions/69094604/npm-fails-during-npm-install-with-npm-err-maximum-call-stack-size-exceeded # npm install on files from host at least sometimes goes down with From 003b55b14a44166e5d366d8ef88f17336203fbc7 Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Wed, 19 Jan 2022 20:45:15 +0100 Subject: [PATCH 09/15] Add one more unexpectedly needed package --- provision/vm_provision.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/provision/vm_provision.sh b/provision/vm_provision.sh index cf1e48790..b979ea998 100644 --- a/provision/vm_provision.sh +++ b/provision/vm_provision.sh @@ -34,6 +34,7 @@ apt-get install -y \ postgresql postgresql-contrib libpq-dev postgis postgresql-12-postgis-3 \ gdal-bin libspatialindex-dev libgeos-dev libproj-dev +apt-get install -y libgdal-dev # TODO: it is not mentioned elsewhere but without it fional fails to install. Investigate. # # Install node.js (node and npm) From e296139c86370fd771a97b3c8b0e92fb4355cf5e Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Wed, 19 Jan 2022 20:46:02 +0100 Subject: [PATCH 10/15] Fix mistake in commented out code --- Vagrantfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 1a0074863..c07e8b2a5 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -30,10 +30,10 @@ Vagrant.configure("2") do |config| # Using VirtualBox: config.vm.provider :virtualbox do |vm| # # Display the VirtualBox GUI when booting the machine - # vb.gui = true + # vm.gui = true # # Customize the amount of memory on the VM: - # vb.memory = "1024" + # vm.memory = "1024" # Enable creating symlinks in shared folder # On a Windows host, vagrant will need to run with permissions to 'Create Symlinks', either From e1cd9611adad8063e7ad02e6ea1b7e285a91b902 Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Wed, 19 Jan 2022 22:52:51 +0100 Subject: [PATCH 11/15] fix typo --- provision/vm_provision.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/provision/vm_provision.sh b/provision/vm_provision.sh index b979ea998..28ff08669 100644 --- a/provision/vm_provision.sh +++ b/provision/vm_provision.sh @@ -34,7 +34,8 @@ apt-get install -y \ postgresql postgresql-contrib libpq-dev postgis postgresql-12-postgis-3 \ gdal-bin libspatialindex-dev libgeos-dev libproj-dev -apt-get install -y libgdal-dev # TODO: it is not mentioned elsewhere but without it fional fails to install. Investigate. +apt-get install -y libgdal-dev # TODO: it is not mentioned elsewhere but without it fiona fails to install. Investigate. + # # Install node.js (node and npm) From ea7a9cae692ca02ecab96b799ae51a4050960642 Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Wed, 19 Jan 2022 22:53:10 +0100 Subject: [PATCH 12/15] reduce pointless noise --- provision/vm_provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/provision/vm_provision.sh b/provision/vm_provision.sh index 28ff08669..d60404c23 100644 --- a/provision/vm_provision.sh +++ b/provision/vm_provision.sh @@ -28,13 +28,13 @@ apt-get update # - python with pip and venv: python3 python3-pip python3-dev # - postgres and postgis: postgresql postgresql-contrib libpq-dev postgis postgresql-12-postgis-3 # - spatial shared libs: gdal-bin libspatialindex-dev libgeos-dev libproj-dev -apt-get install -y \ +apt-get install -y --quiet \ build-essential git vim-nox wget curl \ python3 python3-pip python3-dev python3-venv \ postgresql postgresql-contrib libpq-dev postgis postgresql-12-postgis-3 \ gdal-bin libspatialindex-dev libgeos-dev libproj-dev -apt-get install -y libgdal-dev # TODO: it is not mentioned elsewhere but without it fiona fails to install. Investigate. +apt-get install -y --quiet libgdal-dev # TODO: it is not mentioned elsewhere but without it fiona fails to install. Investigate. # From a0e0dcbba286525526838f37f48d217ee298b1af Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Wed, 19 Jan 2022 22:53:46 +0100 Subject: [PATCH 13/15] Install package that will remove error message on install in the next line --- provision/vm_provision.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/provision/vm_provision.sh b/provision/vm_provision.sh index d60404c23..b45ccfcee 100644 --- a/provision/vm_provision.sh +++ b/provision/vm_provision.sh @@ -108,6 +108,7 @@ source colouringlondon/bin/activate # Install smif pip install --upgrade pip +sudo apt install -y --quiet python3-testresources # to silence an error on install below pip install --upgrade setuptools wheel pip install -r /vagrant/etl/requirements.txt From d2aa6f23c622985c1d8718fd7339502037218ef5 Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Thu, 20 Jan 2022 02:10:02 +0100 Subject: [PATCH 14/15] it is necessary to set owner to protect against failures --- provision/vm_provision.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/provision/vm_provision.sh b/provision/vm_provision.sh index b45ccfcee..55427f128 100644 --- a/provision/vm_provision.sh +++ b/provision/vm_provision.sh @@ -135,6 +135,9 @@ npm install -g npm@6 # TODO: upgrade to later npm to match intended "latest" # which is avoided when remote files are first copied within VM cp /vagrant/app /home/vagrant/ -r +# TODO an ugly hack, would be nice to avoid it +chown -R vagrant:vagrant /home/vagrant/app + # Local fixed install of node modules cd /home/vagrant/app && npm install From 463711317d2b97215af608456fbae76cd643b8a7 Mon Sep 17 00:00:00 2001 From: Mateusz Konieczny Date: Thu, 20 Jan 2022 02:11:59 +0100 Subject: [PATCH 15/15] Increase memory in Vagrant to prevent immediate crashes on startup --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index c07e8b2a5..ba5f5f225 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -33,7 +33,7 @@ Vagrant.configure("2") do |config| # vm.gui = true # # Customize the amount of memory on the VM: - # vm.memory = "1024" + vm.memory = "2048" # default, 1024 was not enough # Enable creating symlinks in shared folder # On a Windows host, vagrant will need to run with permissions to 'Create Symlinks', either