WIP for simple uploader and Mosquitto TLS suppor

.gitignore

@@ -59,3 +59,7 @@ superset/
 # Todo investigate and fix file extension naming inconsistency
 vector-tile-bright-style.json
 .vscode
+docker-compose.override.yml
+conf/mosquitto/start-mosquitto.sh
+conf/mosquitto/mosquitto.conf
+

Makefile

@@ -32,7 +32,7 @@ backup-everything: ## Sequentially run through all backup scripts
 	-@make backup-db-qgis-project
 	@make backup-db
 	@make backup-all-databases
-	@make backup-mergin-base-db-schema
+	-@make backup-mergin-base-db-schema
 	@make backup-node-red
 	@make backup-mosquitto
 	@make backup-jupyter
@@ -69,6 +69,15 @@ deploy: configure ## Deploy the initial stack including nginx, scp and hugo-watc
 	@COMPOSE_PROFILES=$(shell paste -sd, enabled-profiles) docker-compose up -d
 	@COMPOSE_PROFILES=$(shell paste -sd, enabled-profiles) docker-compose logs -f
 
+copy-overrides: ## Copy the docker overrides example if it does not already exist
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Copying overrides"
+	@echo "------------------------------------------------------------------"
+	@if [ -f "docker-compose.override.yml" ]; then echo "Docker composer override already exists."; exit 0; fi
+	@cp docker-compose.override.yml.example docker-compose.override.yml
+
+
 disable-all-services: ## Disable all services - does not actually stop them
 	@echo
 	@echo "------------------------------------------------------------------"
@@ -897,7 +906,7 @@ jupyter-root-shell: ## Create a root bash shell in the jupyter container
 	@COMPOSE_PROFILES=$(shell paste -sd, enabled-profiles) docker-compose exec -u root jupyter bash
 
 
-reinitialise-jupyter:
+restart-jupyter:
 	@make check-env
 	@echo
 	@echo "------------------------------------------------------------------"
@@ -997,7 +1006,7 @@ surveysolutions-root-shell: ## Create a root bash shell in the surveysolutions c
 	@COMPOSE_PROFILES=$(shell paste -sd, enabled-profiles) docker-compose exec -u root surveysolutions bash
 
 
-reinitialise-surveysolutions:
+restart-surveysolutions:
 	@make check-env
 	@echo
 	@echo "------------------------------------------------------------------"
@@ -1328,6 +1337,10 @@ configure-mosquitto:
 	@echo "========================="
 	@echo "Mosquitto configured"
 	@echo "========================="
+	@if [ -f "conf/mosquitto/start-mosquitto.sh" ]; then sudo rm conf/mosquitto/start-mosquitto.sh; fi
+	@cp conf/mosquitto/start-mosquitto.sh.example conf/mosquitto/start-mosquitto.sh
+	@rpl -q {{siteDomain}} $(shell cat conf/nginx_conf/servername.conf | sed 's/         server_name //' | sed 's/;//') conf/mosquitto/start-mosquitto.sh
+	@chmod +x conf/mosquitto/start-mosquitto.sh
 
 restart-mosquitto: stop-mosquitto start-mosquitto
 
@@ -1392,6 +1405,64 @@ restore-mosquitto:
 	@COMPOSE_PROFILES=$(shell paste -sd, enabled-profiles) docker-compose run --entrypoint /bin/bash --rm -w / -v ${PWD}/backups:/backups mosquitto -c "cd /mosquitto/data && tar xvfz /backups/mosquitto-backup.tar.gz --strip 1"
 	@COMPOSE_PROFILES=$(shell paste -sd, enabled-profiles) docker-compose restart mosquitto
 
+#----------------- Simple File Uploader --------------------------
+
+deploy-simple-file-upload: enable-simple-file-upload configure-simple-file-upload start-simple-file-upload 
+
+enable-simple-file-upload:
+	@echo "simple-file-upload" >> enabled-profiles
+
+configure-simple-file-upload:
+	@echo "========================="
+	@echo "Simple file upload configured"
+	@echo "========================="
+	@make copy-overrides
+
+restart-simple-file-upload: stop-simple-file-upload start-simple-file-upload
+
+start-simple-file-upload:
+	@make check-env
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Starting Simple file upload"
+	@echo "------------------------------------------------------------------"
+	@COMPOSE_PROFILES=$(shell paste -sd, enabled-profiles) docker-compose up -d simple-file-upload
+
+stop-simple-file-upload:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Stopping Simple file upload"
+	@echo "------------------------------------------------------------------"
+	-@COMPOSE_PROFILES=$(shell paste -sd, enabled-profiles) docker-compose kill simple-file-upload
+	@COMPOSE_PROFILES=$(shell paste -sd, enabled-profiles) docker-compose rm simple-file-upload
+
+disable-simple-file-upload:
+	@make check-env
+	# Remove from enabled-profiles
+	@sed -i '/simple-file-upload/d' enabled-profiles
+
+simple-file-upload-logs:
+	@make check-env
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Logging simple-file-upload"
+	@echo "------------------------------------------------------------------"
+	@COMPOSE_PROFILES=$(shell paste -sd, enabled-profiles) docker-compose logs -f simple-file-upload
+
+simple-file-upload-shell:
+	@make check-env
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Creating node mosquito shell"
+	@echo "------------------------------------------------------------------"
+	@COMPOSE_PROFILES=$(shell paste -sd, enabled-profiles) docker-compose exec simple-file-upload bash
+
+create-simple-file-upload-user:
+	@echo "------------------------------------------------------------------"
+	@echo "Creating a new simple file upload user"
+	@echo "------------------------------------------------------------------"
+	@export PASSWD=$$(pwgen 20 1); \
+		rpl "# SIMPLE-FILE-UPLOAD-USERS" "# SIMPLE-FILE-UPLOAD-USERS\n      - KEY_$$(PASSWD)=/upload/SomeFile.zip" docker-compose.override.yml 
 
 #----------------- Mergin Server --------------------------
 

conf/mosquitto/mosquitto.conf → conf/mosquitto/mosquitto.conf.example

@@ -5,4 +5,7 @@ allow_anonymous true
 connection_messages true
 #log_dest file /mosquitto/log/mosquitto.log
 listener 1883 0.0.0.0
-
+# For ssl
+cafile /etc/mosquitto/chain.pem
+certfile /etc/mosquitto/cert.pem
+keyfile /etc/mosquitto/privkey.pem

conf/mosquitto/start-mosquitto.sh.example

@@ -0,0 +1,11 @@
+#!/bin/sh
+# This script will require that mosquitto be restarted whenever the cert changes
+mkdir /etc/mosquitto
+# When mosquitto runs it drops down to the mostquitto user
+# and that user needs to be able to access the certs
+# We work on a copy so that we dont end up chaning the permissions globally
+cp /etc/letsencrypt/archive/{{siteDomain}}/chain1.pem /etc/mosquitto/chain.pem
+cp /etc/letsencrypt/archive/{{siteDomain}}/cert1.pem /etc/mosquitto/cert.pem
+cp /etc/letsencrypt/archive/{{siteDomain}}/privkey1.pem /etc/mosquitto/privkey.pem
+chown  mosquitto.mosquitto /etc/mosquitto/*.pem
+mosquitto -c /mosquitto/config/mosquitto.conf

conf/nginx_conf/locations/jupyter.conf.available

@@ -36,7 +36,7 @@
 		proxy_set_header X-Forwarded-Port $server_port;
 
 
-                auth_basic "Jupyter Area";
-                auth_basic_user_file /etc/nginx/.htpasswd; 
+                #auth_basic "Jupyter Area";
+                #auth_basic_user_file /etc/nginx/.htpasswd; 
         }
 

conf/nginx_conf/locations/simple-file-upload.available

@@ -0,0 +1,38 @@
+        location /upload {
+                proxy_pass http://simple-file-upload:3000/upload;
+
+		#Defines the HTTP protocol version for proxying
+		#by default it it set to 1.0.
+		#For Websockets and keepalive connections you need to use the version 1.1    
+		proxy_http_version  1.1;
+
+		#Sets conditions under which the response will not be taken from a cache.
+		proxy_cache_bypass  $http_upgrade;
+
+		#These header fields are required if your application is using Websockets
+		proxy_set_header Upgrade $http_upgrade;
+
+		#These header fields are required if your application is using Websockets    
+		proxy_set_header Connection "upgrade";
+
+		#The $host variable in the following order of precedence contains:
+		#hostname from the request line, or hostname from the Host request header field
+		#or the server name matching a request.    
+		proxy_set_header Host $host;
+
+		#Forwards the real visitor remote IP address to the proxied server
+		proxy_set_header X-Real-IP $remote_addr;
+
+		#A list containing the IP addresses of every server the client has been proxied through    
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+		#When used inside an HTTPS server block, each HTTP response from the proxied server is rewritten to HTTPS.    
+		proxy_set_header X-Forwarded-Proto $scheme;
+
+		#Defines the original host requested by the client.    
+		proxy_set_header X-Forwarded-Host $host;
+
+		#Defines the original port requested by the client.    
+		proxy_set_header X-Forwarded-Port $server_port;
+        }
+

conf/nginx_conf/locations/surveysolutions.conf.available

@@ -0,0 +1,38 @@
+        location /survey/ {
+                proxy_pass http://surveysolutions/;
+
+		#Defines the HTTP protocol version for proxying
+		#by default it it set to 1.0.
+		#For Websockets and keepalive connections you need to use the version 1.1    
+		proxy_http_version  1.1;
+
+		#Sets conditions under which the response will not be taken from a cache.
+		proxy_cache_bypass  $http_upgrade;
+
+		#These header fields are required if your application is using Websockets
+		proxy_set_header Upgrade $http_upgrade;
+
+		#These header fields are required if your application is using Websockets    
+		proxy_set_header Connection "upgrade";
+
+		#The $host variable in the following order of precedence contains:
+		#hostname from the request line, or hostname from the Host request header field
+		#or the server name matching a request.    
+		proxy_set_header Host $host;
+
+		#Forwards the real visitor remote IP address to the proxied server
+		proxy_set_header X-Real-IP $remote_addr;
+
+		#A list containing the IP addresses of every server the client has been proxied through    
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+		#When used inside an HTTPS server block, each HTTP response from the proxied server is rewritten to HTTPS.    
+		proxy_set_header X-Forwarded-Proto $scheme;
+
+		#Defines the original host requested by the client.    
+		proxy_set_header X-Forwarded-Host $host;
+
+		#Defines the original port requested by the client.    
+		proxy_set_header X-Forwarded-Port $server_port;
+        }
+

docker-compose.yml

@@ -32,8 +32,10 @@ x-lizmap-image: &lizmap-image 3liz/lizmap-web-client:3.4
 # Auto builds on the main branch are tagged with :main
 x-hugo-image: &hugo-image kartoza/hugo-watcher:main
 x-node-red-image: &node-red-image nodered/node-red:latest
-x-mosquitto-image: &mosquitto-image eclipse-mosquitto
+x-mosquitto-image: &mosquitto-image eclipse-mosquitto:openssl
 x-surveysolutions-image: &surveysolutions-image surveysolutions/surveysolutions
+x-simple-file-upload-image: &simple-file-upload-image twostoryrobot/simple-file-upload
+
 # 
 # VOLUMES
 #
@@ -201,6 +203,30 @@ services:
      - ./nginx_certbot_init_conf/nginx.conf:/etc/nginx/nginx.conf
      - ./certbot/certbot/conf:/etc/letsencrypt
      - ./certbot/certbot/www:/var/www/certbot
+
+  # file uploader
+  simple-file-upload:
+    image: *simple-file-upload-image
+    profiles: ["all","simple-file-upload"]
+    volumes:
+     # General sharing of files to the web
+     # You can also upload using the scp container
+     - scp_general_data:/upload
+    # Please see docker-compose.override.yml which automatically
+    # gets imported when you run docker-compose
+    ports:
+      - "3000:3000"
+    networks:
+     - os-gis-stack
+    restart: unless-stopped
+    logging:
+      driver: json-file
+      options:
+        max-size: 200m
+        max-file: '10'
+    depends_on:
+      - scp
+      - nginx
 
   hugo-watcher:
     # This service watches for changes in the static content
@@ -744,7 +770,10 @@ services:
       - os-gis-stack  
     volumes:
       - mosquitto-data:/mosquitto/data
+      - ./certbot/certbot/conf:/etc/letsencrypt
       - ./conf/mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf 
+      - ./conf/mosquitto/start-mosquitto.sh:/mosquitto/config/start-mosquitto.sh
+    command: /mosquitto/config/start-mosquitto.sh
     logging:
       driver: json-file
       options:
@@ -1318,7 +1347,7 @@ services:
     profiles: 
       - surveysolutions
     ports:
-     - 80
+      - 9999:80
     depends_on:
      - db
     networks: