diff --git a/.gitignore b/.gitignore index 56060de36..c523cb808 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ etl/images/* etl/*.geojson etl/**/*.csv etl/**/*.txt +!etl/**/requirements.txt etl/**/*.xls etl/**/*.xlsx etl/**/*.zip diff --git a/Vagrantfile b/Vagrantfile index 181e3aec6..ba5f5f225 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -12,7 +12,7 @@ Vagrant.configure("2") do |config| # Every Vagrant development environment requires a box. You can search for # boxes at https://vagrantcloud.com/search. - config.vm.box = "bento/ubuntu-18.04" + config.vm.box = "ubuntu/focal64" # Create a forwarded port mapping which allows access to a specific port # within the machine from a port on the host machine and only allow access @@ -30,10 +30,10 @@ Vagrant.configure("2") do |config| # Using VirtualBox: config.vm.provider :virtualbox do |vm| # # Display the VirtualBox GUI when booting the machine - # vb.gui = true + # vm.gui = true # # Customize the amount of memory on the VM: - # vb.memory = "1024" + vm.memory = "2048" # default, 1024 was not enough # Enable creating symlinks in shared folder # On a Windows host, vagrant will need to run with permissions to 'Create Symlinks', either diff --git a/etl/requirements.txt b/etl/requirements.txt index d1e037461..797c78e95 100644 --- a/etl/requirements.txt +++ b/etl/requirements.txt @@ -1,7 +1,7 @@ # Python packages for etl fiona==1.7.13 osmnx==0.8.1 -psycopg2==2.7.5 +psycopg2==2.8.4 # test is it working, used to be 2.7.5 (no obviously borken things) shapely==1.6.4 retrying==1.3.3 requests==2.23.0 diff --git a/maintenance/requirements.txt b/maintenance/requirements.txt index ffa77af7b..7a98cb02a 100644 --- a/maintenance/requirements.txt +++ b/maintenance/requirements.txt @@ -1,2 +1,2 @@ -psycopg2==2.8.3 -requests==2.24.0 \ No newline at end of file +psycopg2==2.8.4 +requests==2.24.0 diff --git a/provision/vm_provision.sh b/provision/vm_provision.sh index 88f08d70a..55427f128 100644 --- a/provision/vm_provision.sh +++ b/provision/vm_provision.sh @@ -28,12 +28,14 @@ apt-get update # - python with pip and venv: python3 python3-pip python3-dev # - postgres and postgis: postgresql postgresql-contrib libpq-dev postgis postgresql-12-postgis-3 # - spatial shared libs: gdal-bin libspatialindex-dev libgeos-dev libproj-dev -apt-get install -y \ +apt-get install -y --quiet \ build-essential git vim-nox wget curl \ python3 python3-pip python3-dev python3-venv \ postgresql postgresql-contrib libpq-dev postgis postgresql-12-postgis-3 \ gdal-bin libspatialindex-dev libgeos-dev libproj-dev +apt-get install -y --quiet libgdal-dev # TODO: it is not mentioned elsewhere but without it fiona fails to install. Investigate. + # # Install node.js (node and npm) @@ -73,9 +75,9 @@ service postgresql start # Ensure en_US locale exists locale-gen en_US.UTF-8 # Database config to listen on network connection -sed -i "s/#\?listen_address.*/listen_addresses '*'/" /etc/postgresql/10/main/postgresql.conf +sed -i "s/#\?listen_address.*/listen_addresses '*'/" /etc/postgresql/12/main/postgresql.conf # Allow password connections from any IP (so includes host) -echo "host all all all md5" >> /etc/postgresql/10/main/pg_hba.conf +echo "host all all all md5" >> /etc/postgresql/12/main/pg_hba.conf # Restart postgres to pick up config changes service postgresql restart @@ -106,6 +108,7 @@ source colouringlondon/bin/activate # Install smif pip install --upgrade pip +sudo apt install -y --quiet python3-testresources # to silence an error on install below pip install --upgrade setuptools wheel pip install -r /vagrant/etl/requirements.txt @@ -117,11 +120,26 @@ chown -R vagrant:vagrant /home/vagrant/colouringlondon # Install node modules # -# Install latest release of npm -npm install -g npm@next +# Install release v6 of npm +npm install -g npm@6 # TODO: upgrade to later npm to match intended "latest" + +# https://stackoverflow.com/questions/69094604/npm-fails-during-npm-install-with-npm-err-maximum-call-stack-size-exceeded +# npm install on files from host at least sometimes goes down with +# +# 957 verbose stack RangeError: Maximum call stack size exceeded +# 957 verbose stack at RegExp.test () +# 957 verbose stack at isDepOptional (/usr/local/lib/node/node-v12.18.1/lib/node_modules/npm/lib/install/deps.js:432:45) +# 957 verbose stack at failedDependency (/usr/local/lib/node/node-v12.18.1/lib/node_modules/npm/lib/install/deps.js:441:9) +# (..) +# +# which is avoided when remote files are first copied within VM +cp /vagrant/app /home/vagrant/ -r + +# TODO an ugly hack, would be nice to avoid it +chown -R vagrant:vagrant /home/vagrant/app # Local fixed install of node modules -cd /vagrant/app && npm install +cd /home/vagrant/app && npm install #