From fdf5b188ce516a319f96a664052122b4d3bb4a24 Mon Sep 17 00:00:00 2001 From: Sietse Snel Date: Tue, 8 Oct 2024 10:52:48 +0200 Subject: [PATCH] YDA-5949: Add Docker setup for development (Work in progress) --- .ansible-lint | 3 + docker/.env | 47 + docker/README.md | 26 + docker/build-local-images.sh | 13 + docker/docker-compose.yml | 74 ++ docker/images/ckan/Dockerfile | 106 ++ docker/images/ckan/build.sh | 2 + docker/images/ckan/ckan-entrypoint.sh | 65 ++ docker/images/ckan/ckan-uwsgi.ini | 15 + docker/images/ckan/ckan.ini | 274 +++++ docker/images/ckan/who.ini | 32 + docker/images/ckan/wsgi.py | 12 + docker/images/nginx/Dockerfile | 11 + docker/images/nginx/build.sh | 2 + docker/images/nginx/ckan.site | 55 + docker/images/nginx/epos-msl.cnf | 13 + docker/images/nginx/nginx-entrypoint.sh | 24 + docker/images/solr/Dockerfile | 27 + docker/images/solr/build.sh | 2 + docker/images/solr/solrconfig.xml | 1360 +++++++++++++++++++++++ 20 files changed, 2163 insertions(+) create mode 100644 docker/.env create mode 100644 docker/README.md create mode 100755 docker/build-local-images.sh create mode 100644 docker/docker-compose.yml create mode 100644 docker/images/ckan/Dockerfile create mode 100755 docker/images/ckan/build.sh create mode 100644 docker/images/ckan/ckan-entrypoint.sh create mode 100644 docker/images/ckan/ckan-uwsgi.ini create mode 100644 docker/images/ckan/ckan.ini create mode 100644 docker/images/ckan/who.ini create mode 100644 docker/images/ckan/wsgi.py create mode 100644 docker/images/nginx/Dockerfile create mode 100755 docker/images/nginx/build.sh create mode 100644 docker/images/nginx/ckan.site create mode 100644 docker/images/nginx/epos-msl.cnf create mode 100644 docker/images/nginx/nginx-entrypoint.sh create mode 100644 docker/images/solr/Dockerfile create mode 100755 docker/images/solr/build.sh create mode 100644 docker/images/solr/solrconfig.xml diff --git a/.ansible-lint b/.ansible-lint index 27fb9d4..8d34551 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -1,3 +1,6 @@ +exclude_paths: + - ./.github + - ./docker skip_list: - no-changed-when # Commands should not change things if nothing needs doing - no-handler # Tasks that run when changed should likely be handlers diff --git a/docker/.env b/docker/.env new file mode 100644 index 0000000..556377e --- /dev/null +++ b/docker/.env @@ -0,0 +1,47 @@ +# Variables in this file will be substituted into docker-compose.yml +# Save a copy of this file as .env and insert your own values. +# Verify correct substitution with "docker-compose config" +# If variables are newly added or enabled, please delete and rebuild the images to pull in changes: +# docker-compose down +# docker rmi -f docker_ckan docker_db +# docker rmi $(docker images -f dangling=true -q) +# docker-compose build +# docker-compose up -d +# docker-compose restart ckan # give the db service time to initialize the db cluster on first run + +# Image: ckan +CKAN_SITE_ID=default + +EPOS_MSL_HOST=epos-msl.ckan +# +# On AWS, your CKAN_SITE_URL is the output of: +# curl -s http://169.254.169.254/latest/meta-data/public-hostname +# CKAN_SITE_URL=http://ec2-xxx-xxx-xxx-xxx.ap-southeast-2.compute.amazonaws.com +# On OSX with Docker for Mac, your CKAN_SITE_URL is +# CKAN_SITE_URL=http://docker.for.mac.localhost:5000 +# When running locally, CKAN_SITE_URL must contain the port +CKAN_SITE_URL=http://localhost:5000 +# +# CKAN_PORT must be available on the host: sudo netstat -na +# To apply change: docker-compose down && docker rmi docker_ckan && docker-compose build ckan +CKAN_PORT=5000 +# +# Email settings +CKAN_SMTP_SERVER=smtp.corporateict.domain:25 +CKAN_SMTP_STARTTLS=True +CKAN_SMTP_USER=user +CKAN_SMTP_PASSWORD=pass +CKAN_SMTP_MAIL_FROM=ckan@localhost +# +# Image: db +POSTGRES_PASSWORD=ckan +# +# POSTGRES_PORT must be available on the host: sudo netstat -na | grep 5432 +# To apply change: docker-compose down && docker rmi docker_db docker_ckan && docker-compose build +POSTGRES_PORT=5432 +# +# The datastore database will be created in the db container as docs +# Readwrite user/pass will be ckan:POSTGRES_PASSWORD +# Readonly user/pass will be datastore_ro:DATASTORE_READONLY_PASSWORD +DATASTORE_READONLY_PASSWORD=datastore + diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..b7606c4 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,26 @@ +# EPOS-MSL Docker development setup + +## Building the images + +The images of the EPOS-MSL are not yet available in a registry, so you'll have to build them locally first. + +``` +./build-local-images.sh +``` + +## Using the Docker setup + +First add an entry to your `/etc/hosts` file (or equivalent) so that queries for the development setup +interface resolve to your loopback interface. For example: + +``` +127.0.0.1 epos-msl.ckan +``` + +Start the Docker Compose setup: +``` +docker compose up +``` + +Wait until CKAN has started. Then navigate to [https://epos-msl.ckan](https://epos-msl.ckan) in your browser. The +development VM runs with self-signed certificates, so you'll need to accept the security warning. diff --git a/docker/build-local-images.sh b/docker/build-local-images.sh new file mode 100755 index 0000000..a93e8f5 --- /dev/null +++ b/docker/build-local-images.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -e +set -x + +cd images/ckan +./build.sh +cd ../nginx +./build.sh +cd ../solr +./build.sh + +set +x +echo "Building images completed." diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..56cfaf2 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,74 @@ +# This is a locally modified version of the upstream CKAN docker compose file. +# It contains image names for pushing locally built images to a central registry at ECR + +# docker-compose build && docker-compose up -d +# If "docker-compose logs ckan" shows DB not ready, run "docker-compose restart ckan" a few times. +version: "3" + +volumes: + ckan_config: + ckan_home: + ckan_storage: + ckan_coveragedata: + pg_data: + solr_data: + nginx_config: + nginx_certificates: + +services: + ckan: + container_name: ckan + image: epos-msl-ckan:latest + links: + - db + - solr + - redis + depends_on: + - db + - solr + - redis + ports: + - "0.0.0.0:${CKAN_PORT}:5000" + environment: + - CKAN_SQLALCHEMY_URL=postgresql://ckan:${POSTGRES_PASSWORD}@db/ckan_default + - CKAN_SOLR_URL=http://solr:8983/solr/ckan + - CKAN_REDIS_URL=redis://redis:6379/1 + - CKAN_SITE_URL=${CKAN_SITE_URL} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + volumes: + - ckan_config:/etc/ckan + - ckan_home:/usr/lib/ckan + - ckan_storage:/var/lib/ckan + - ckan_coveragedata:/coverage + + nginx: + container_name: nginx + image: epos-msl-nginx:latest + environment: + - EPOS_MSL_HOST=${EPOS_MSL_HOST} + ports: + - "18443:443" + volumes: + - nginx_config:/etc/nginx/conf.d + - nginx_certificates:/etc/certificates + + db: + container_name: db + image: postgres:12.20 + environment: + - POSTGRES_DB=ckan_default + - POSTGRES_USER=ckan + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - PGDATA=/var/lib/postgresql/data/db + volumes: + - pg_data:/var/lib/postgresql/data + + solr: + container_name: solr + image: epos-msl-solr:latest + volumes: + - solr_data:/opt/solr/server/solr/ckan/data + + redis: + container_name: redis + image: redis:6.2 diff --git a/docker/images/ckan/Dockerfile b/docker/images/ckan/Dockerfile new file mode 100644 index 0000000..de6d49e --- /dev/null +++ b/docker/images/ckan/Dockerfile @@ -0,0 +1,106 @@ +# This is a locally modified version of the upstream CKAN Dockerfile + +# See CKAN docs on installation from Docker Compose on usage +FROM ubuntu:focal-20210119 +MAINTAINER Yoda team + +# Set timezone +ENV TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Setting the locale +ENV LC_ALL=en_US.UTF-8 +RUN adduser ckan +RUN apt-get update +RUN apt-get install --no-install-recommends -y locales +RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen +RUN dpkg-reconfigure --frontend=noninteractive locales +RUN update-locale LANG=${LC_ALL} + +# Install required system packages +RUN apt-get -q -y update \ + && DEBIAN_FRONTEND=noninteractive apt-get -q -y upgrade \ + && apt-get -q -y install \ + python3.8 \ + python3-dev \ + python3-pip \ + python3-venv \ + python3-wheel \ + libpq-dev \ + python3-pastescript \ + python3-virtualenv \ + libxml2-dev \ + libxslt-dev \ + libgeos-dev \ + libssl-dev \ + libffi-dev \ + postgresql-client \ + build-essential \ + git-core \ + vim \ + wget \ + curl \ + nmap \ + sqlite3 \ + pwgen \ + uuid-runtime \ + && apt-get -q clean \ + && rm -rf /var/lib/apt/lists/* + +# Define environment variables +ENV CKAN_HOME /usr/lib/ckan +ENV CKAN_VENV $CKAN_HOME/default +ENV CKAN_CONFIG /etc/ckan +ENV CKAN_STORAGE_PATH=/ckanstorage +ENV CKAN_VERSION=2.9.11 +ENV CKAN_SCHEMING_VERSION=release-2.1.0 +ENV CKAN_MSL_CORE_VERSION=1.4.0 +ENV CKAN_MSL_UTIL_VERSION=1.0.0 + +# Create storage path +RUN mkdir -p $CKAN_STORAGE_PATH/webassets $CKAN_STORAGE_PATH/storage + +# Build-time variables specified by docker-compose.yml / .env +ARG CKAN_SITE_URL + +# Setup virtual environment for CKAN +RUN mkdir -p $CKAN_VENV $CKAN_CONFIG/default && \ + python3 -m venv $CKAN_VENV && \ + ln -s $CKAN_VENV/bin/pip3 /usr/local/bin/ckan-pip3 &&\ + ln -s $CKAN_VENV/bin/ckan /usr/local/bin/ckan +ADD ckan.ini /etc/ckan/default/ckan.ini +ADD who.ini /etc/ckan/default/who.ini +ADD wsgi.py /etc/ckan/default/wsgi.py +ADD ckan-uwsgi.ini /etc/ckan/default/ckan-uwsgi.ini + +# Virtual environment binaries/scripts to be used first +ENV PATH=${CKAN_VENV}/bin:${PATH} +ENV EPOS_MSL_FQDN=epos-msl.local +ENV CKAN_ADMIN_PASSWORD="testtest" + +# Install CKAN and plugins +RUN ckan-pip3 install -U pip && \ + ckan-pip3 install setuptools==44.1.0 && \ + ckan-pip3 install --upgrade pip && \ + ckan-pip3 install wheel && \ + ckan-pip3 install -e "git+https://github.com/ckan/ckan@ckan-${CKAN_VERSION}#egg=ckan[requirements]" && \ + ckan-pip3 install uwsgi && \ + ckan-pip3 install -e "git+https://github.com/ckan/ckanext-scheming@${CKAN_SCHEMING_VERSION}#egg=ckanext-scheming" && \ + ckan-pip3 install -e "git+https://github.com/UtrechtUniversity/msl_ckan_core@${CKAN_MSL_CORE_VERSION}#egg=ckanext-msl_ckan" && \ + ckan-pip3 install -e "git+https://github.com/UtrechtUniversity/msl_ckan_util@${CKAN_MSL_UTIL_VERSION}#egg=ckanext-msl_ckan_util" && \ + ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \ + cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \ + chmod +x /ckan-entrypoint.sh && \ + chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH $COVERAGE_DIR && \ + rm /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml && \ + ln -sf /usr/lib/ckan/default/src/ckanext-msl-ckan/ckanext/msl_ckan/config/solr/schema.xml /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml && \ + perl -n -i.bak -e 'print unless /defaultSearchField/ or /solrQueryParser/' /usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml + +ADD ./ckan-entrypoint.sh /ckan-entrypoint.sh +ENTRYPOINT ["/ckan-entrypoint.sh"] +RUN chmod +x /ckan-entrypoint.sh + +USER ckan +EXPOSE 8080 + +CMD ["/ckan-entrypoint.sh"] diff --git a/docker/images/ckan/build.sh b/docker/images/ckan/build.sh new file mode 100755 index 0000000..9712778 --- /dev/null +++ b/docker/images/ckan/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker build -t epos-msl-ckan:latest . diff --git a/docker/images/ckan/ckan-entrypoint.sh b/docker/images/ckan/ckan-entrypoint.sh new file mode 100644 index 0000000..53dfa4e --- /dev/null +++ b/docker/images/ckan/ckan-entrypoint.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +set -u + +check_port() { + local host="$1" + local port="$2" + + echo "Checking service availability at ${host}:${port}..." + + while true; do + # Use nmap to scan the specified port on the given host + nmap -p "${port}" "${host}" | grep -q "open" + + # If the port is open, exit the loop + if [ $? -eq 0 ]; then + echo "Server is available at ${host}:${port}" + break + else + echo "Server at ${host}:${port} is not available. Retrying in 1 seconds..." + sleep 1 + fi + done +} + +## Check DB up +check_port db 5432 + +## Check Solr up +check_port solr 8983 + +## Check Redis up +check_port redis 6379 + +## Initialize CKAN config, database and admin account +CKAN_CONFIG_FILE=/etc/ckan/default/ckan.ini +CKAN_INIT_STATUS_FILE=/etc/ckan/default/.ckan_initialized + +if test -f "$CKAN_INIT_STATUS_FILE" +then echo "Configuration and database already initialized." +else echo "Initializing configuration ..." + export BEAKER_SESSION_SECRET=$(openssl rand -base64 32) + export APP_INSTANCE_UUID=$(uuidgen --name "$EPOS_MSL_FQDN" --namespace "@url" --sha1) + export CKAN_DATABASE_PASSWORD=$(pwgen -n 16 -N 1) + export CKAN_MSL_VOCABULARIES_ENDPOINT="https://${EPOS_MSL_FQDN}/webservice/api/vocabularies" + perl -pi.bak -e '$beaker_session_secret=$ENV{BEAKER_SESSION_SECRET}; s/BEAKER_SESSION_SECRET/$beaker_session_secret/ge' "$CKAN_CONFIG_FILE" + perl -pi.bak -e '$app_instance_uuid=$ENV{APP_INSTANCE_UUID}; s/APP_INSTANCE_UUID/$app_instance_uuid/ge' "$CKAN_CONFIG_FILE" + perl -pi.bak -e '$ckan_database_password=$ENV{CKAN_DATABASE_PASSWORD}; s/CKAN_DATABASE_PASSWORD/$ckan_database_password/ge' "$CKAN_CONFIG_FILE" + perl -pi.bak -e '$epos_msl_fqdn=$ENV{EPOS_MSL_FQDN}; s/EPOS_MSL_FQDN/$epos_msl_fqdn/ge' "$CKAN_CONFIG_FILE" + perl -pi.bak -e '$ckan_msl_vocabularies_endpoint=$ENV{CKAN_MSL_VOCABULARIES_ENDPOINT}; s/CKAN_MSL_VOCABULARIES_ENDPOINT/$ckan_msl_vocabularies_endpoint/ge' "$CKAN_CONFIG_FILE" + echo "Initializing database ..." + /usr/lib/ckan/default/bin/ckan -c "$CKAN_CONFIG_FILE" db init + /usr/lib/ckan/default/bin/ckan -c "$CKAN_CONFIG_FILE" user add ckanadmin password="$CKAN_ADMIN_PASSWORD" email=ckanadmin@localhost name=ckanadmin + /usr/lib/ckan/default/bin/ckan -c "$CKAN_CONFIG_FILE" sysadmin add ckanadmin + touch "$CKAN_INIT_STATUS_FILE" + echo "Configuration and database initialization finished." +fi + +## Start CKAN +echo "Starting CKAN ..." +while true +do /usr/lib/ckan/default/bin/uwsgi -i /etc/ckan/default/ckan-uwsgi.ini + sleep 1 + echo "CKAN process terminated; trying to restart ..." +done diff --git a/docker/images/ckan/ckan-uwsgi.ini b/docker/images/ckan/ckan-uwsgi.ini new file mode 100644 index 0000000..d30aa91 --- /dev/null +++ b/docker/images/ckan/ckan-uwsgi.ini @@ -0,0 +1,15 @@ +[uwsgi] + +http = 0.0.0.0:8080 +uid = www-data +gid = www-data +wsgi-file = /etc/ckan/default/wsgi.py +virtualenv = /usr/lib/ckan/default +module = wsgi:application +master = true +pidfile = /tmp/%n.pid +harakiri = 50 +max-requests = 5000 +vacuum = true +callable = application +strict = true diff --git a/docker/images/ckan/ckan.ini b/docker/images/ckan/ckan.ini new file mode 100644 index 0000000..c1f591b --- /dev/null +++ b/docker/images/ckan/ckan.ini @@ -0,0 +1,274 @@ +# +# CKAN - Pylons configuration +# +# These are some of the configuration options available for your CKAN +# instance. Check the documentation in 'doc/configuration.rst' or at the +# following URL for a description of what they do and the full list of +# available options: +# +# http://docs.ckan.org/en/latest/maintaining/configuration.html +# +# The %(here)s variable will be replaced with the parent directory of this file +# + +[DEFAULT] + +# WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PUBLIC ENVIRONMENT* +# With debug mode enabled, a visitor to your site could execute malicious commands. +debug = false + +[app:main] +use = egg:ckan + +## Development settings +ckan.devserver.host = localhost +ckan.devserver.port = 5000 + +## Session settings +cache_dir = /tmp/%(ckan.site_id)s/ +beaker.session.key = ckan + +# This is the secret token that the beaker library uses to hash the cookie sent +# to the client. `ckan generate config` generates a unique value for this each +# time it generates a config file. +beaker.session.secret = BEAKER_SESSION_SECRET + +# `ckan generate config` generates a unique value for this each time it generates +# a config file. +app_instance_uuid = APP_INSTANCE_UUID + +# Default number of search facets returned in a query +search.facets.limit = 10000 + +# repoze.who config +who.config_file = %(here)s/who.ini +who.log_level = warning +who.log_file = %(cache_dir)s/who_log.ini +# Session timeout (user logged out after period of inactivity, in seconds). +# Inactive by default, so the session doesn't expire. +who.timeout = 3600 + +## Database Settings +sqlalchemy.url = postgresql://ckan:CKAN_DATABASE_PASSWORD@db/ckan_default + +#ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default +#ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default + +# PostgreSQL' full-text search parameters +ckan.datastore.default_fts_lang = english +ckan.datastore.default_fts_index_method = gist + + +## Site Settings + +ckan.site_url = https://EPOS_MSL_FQDN +#ckan.use_pylons_response_cleanup_middleware = true + +# Default timeout for Requests +#ckan.requests.timeout = 10 + + +## Authorization Settings + +ckan.auth.anon_create_dataset = false +ckan.auth.create_unowned_dataset = false +ckan.auth.create_dataset_if_not_in_organization = false +ckan.auth.user_create_groups = false +ckan.auth.user_create_organizations = false +ckan.auth.user_delete_groups = true +ckan.auth.user_delete_organizations = true +ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = false +ckan.auth.roles_that_cascade_to_sub_groups = admin +ckan.auth.public_user_details = false +ckan.auth.public_activity_stream_detail = false +ckan.auth.allow_dataset_collaborators = false +ckan.auth.create_default_api_keys = false + +## API Token Settings +api_token.nbytes = 60 +api_token.jwt.encode.secret = string:HvlXPoquK7PcVZ1eMo7Lwf_Rs +api_token.jwt.decode.secret = string:HvlXPoquK7PcVZ1eMo7Lwf_Rs +api_token.jwt.algorithm = HS256 + +## API Token: expire_api_token plugin +expire_api_token.default_lifetime = 3600 + +## Search Settings + +ckan.site_id = default + +solr_url = http://solr:8983/solr/ckan + +## Redis Settings + +# URL to your Redis instance, including the database to be used. +ckan.redis.url = redis://redis:6379/0 + + +## CORS Settings + +# If cors.origin_allow_all is true, all origins are allowed. +# If false, the cors.origin_whitelist is used. +# ckan.cors.origin_allow_all = true +# cors.origin_whitelist is a space separated list of allowed domains. +# ckan.cors.origin_whitelist = http://example1.com http://example2.com + + +## Plugins Settings + +# Note: Add ``datastore`` to enable the CKAN DataStore +# Add ``datapusher`` to enable DataPusher +# Add ``resource_proxy`` to enable resorce proxying and get around the +# same origin policy +#ckan.plugins = stats text_view image_view recline_view +ckan.plugins = stats text_view image_view recline_view msl_ckan scheming_datasets scheming_groups scheming_organizations msl_custom_facets msl_repeating_fields + +scheming.dataset_schemas = ckanext.msl_ckan:schemas/datasets/data_publication.yml ckanext.msl_ckan:schemas/datasets/labs.json +scheming.organization_schemas = ckanext.msl_ckan:schemas/organizations/organization.json + +mslfacets.dataset_config = ckanext.msl_ckan:config/facets.json + +mslindexfields.field_config = ckanext.msl_ckan:config/msl_index_fields.json + +mslvocabularies.endpoint_root = CKAN_MSL_VOCABULARIES_ENDPOINT + +# Define which views should be created by default +# (plugins must be loaded in ckan.plugins) +ckan.views.default_views = image_view text_view recline_view + +# Customize which text formats the text_view plugin will show +#ckan.preview.json_formats = json +#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss +#ckan.preview.text_formats = text plain text/plain + +# Customize which image formats the image_view plugin will show +#ckan.preview.image_formats = png jpeg jpg gif + +## Front-End Settings + +ckan.site_title = CKAN +ckan.site_logo = /base/images/ckan-logo.png +ckan.site_description = +ckan.favicon = /base/images/ckan.ico +ckan.gravatar_default = identicon +ckan.preview.direct = png jpg gif +ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json +ckan.display_timezone = server + +# package_hide_extras = for_search_index_only +#package_edit_return_url = http://another.frontend/dataset/ +#package_new_return_url = http://another.frontend/dataset/ +#ckan.recaptcha.publickey = +#ckan.recaptcha.privatekey = +#licenses_group_url = http://licenses.opendefinition.org/licenses/groups/ckan.json +# ckan.template_footer_end = + + +## Internationalisation Settings +ckan.locale_default = en +ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv +ckan.locales_offered = +ckan.locales_filtered_out = en_GB + +## Feeds Settings + +ckan.feeds.authority_name = +ckan.feeds.date = +ckan.feeds.author_name = +ckan.feeds.author_link = + +## Storage Settings + +ckan.storage_path = /ckanstorage +#ckan.max_resource_size = 10 +#ckan.max_image_size = 2 + +## Webassets Settings +#ckan.webassets.use_x_sendfile = false +#ckan.webassets.path = /var/lib/ckan/webassets + + +## Datapusher settings + +# Make sure you have set up the DataStore + +#ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +#ckan.datapusher.url = http://127.0.0.1:8800/ +#ckan.datapusher.assume_task_stale_after = 3600 + +# Resource Proxy settings +# Preview size limit, default: 1MB +#ckan.resource_proxy.max_file_size = 1048576 +# Size of chunks to read/write. +#ckan.resource_proxy.chunk_size = 4096 +# Default timeout for fetching proxied items +#ckan.resource_proxy.timeout = 10 + +## Activity Streams Settings + +#ckan.activity_streams_enabled = true +#ckan.activity_list_limit = 31 +#ckan.activity_streams_email_notifications = true +#ckan.email_notifications_since = 2 days +ckan.hide_activity_from_users = %(ckan.site_id)s + + +## Email settings + +#email_to = errors@example.com +#error_email_from = ckan-errors@example.com +#smtp.server = localhost +#smtp.starttls = False +#smtp.user = username@example.com +#smtp.password = your_password +#smtp.mail_from = +#smtp.reply_to = + +## Background Job Settings +ckan.jobs.timeout = 180 + +## Logging configuration +[loggers] +keys = root, ckan, ckanext, werkzeug + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_werkzeug] +level = WARNING +handlers = console +qualname = werkzeug +propagate = 0 + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s + +# msl_ckan_util settings +ckan.mslfacets.dataset_config = ckanext.msl_ckan_util:samples/facets.json +ckan.mslindexfields.fields_config = ckanext.msl_ckan_util:samples/msl_index_fields.json diff --git a/docker/images/ckan/who.ini b/docker/images/ckan/who.ini new file mode 100644 index 0000000..eb2a5a5 --- /dev/null +++ b/docker/images/ckan/who.ini @@ -0,0 +1,32 @@ +[plugin:auth_tkt] +use = ckan.lib.repoze_plugins.auth_tkt:make_plugin +# If no secret key is defined here, beaker.session.secret will be used +#secret = somesecret + +[plugin:friendlyform] +use = ckan.lib.repoze_plugins.friendly_form:FriendlyFormPlugin +login_form_url= /user/login +login_handler_path = /login_generic +logout_handler_path = /user/logout +rememberer_name = auth_tkt +post_login_url = /user/logged_in +post_logout_url = /user/logged_out +charset = utf-8 + +[general] +request_classifier = repoze.who.classifiers:default_request_classifier +challenge_decider = repoze.who.classifiers:default_challenge_decider + +[identifiers] +plugins = + friendlyform;browser + auth_tkt + +[authenticators] +plugins = + auth_tkt + ckan.lib.authenticator:UsernamePasswordAuthenticator + +[challengers] +plugins = + friendlyform;browser diff --git a/docker/images/ckan/wsgi.py b/docker/images/ckan/wsgi.py new file mode 100644 index 0000000..9880af9 --- /dev/null +++ b/docker/images/ckan/wsgi.py @@ -0,0 +1,12 @@ +# -- coding: utf-8 -- + +import os +from ckan.config.middleware import make_app +from ckan.cli import CKANConfigLoader +from logging.config import fileConfig as loggingFileConfig +config_filepath = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'ckan.ini') +abspath = os.path.join(os.path.dirname(os.path.abspath(__file__))) +loggingFileConfig(config_filepath) +config = CKANConfigLoader(config_filepath).get_config() +application = make_app(config) diff --git a/docker/images/nginx/Dockerfile b/docker/images/nginx/Dockerfile new file mode 100644 index 0000000..13ccae5 --- /dev/null +++ b/docker/images/nginx/Dockerfile @@ -0,0 +1,11 @@ +# Image for Nginx reverse proxy configured for CKAN + +FROM nginx:1.27 +MAINTAINER Yoda team + +ADD ckan.site /etc/nginx/conf.d/ckan.conf +RUN rm /etc/nginx/conf.d/default.conf +ADD epos-msl.cnf /etc/certificates/epos-msl.cnf +ADD nginx-entrypoint.sh /nginx-entrypoint.sh +RUN chmod +x /nginx-entrypoint.sh +CMD ["/nginx-entrypoint.sh"] diff --git a/docker/images/nginx/build.sh b/docker/images/nginx/build.sh new file mode 100755 index 0000000..4c015cc --- /dev/null +++ b/docker/images/nginx/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker build -t epos-msl-nginx:latest . diff --git a/docker/images/nginx/ckan.site b/docker/images/nginx/ckan.site new file mode 100644 index 0000000..b6a2ede --- /dev/null +++ b/docker/images/nginx/ckan.site @@ -0,0 +1,55 @@ +proxy_cache_path /tmp/nginx_cache levels=1:2 keys_zone=cache:30m max_size=250m; +proxy_temp_path /tmp/nginx_proxy 1 2; + +server { + listen 80 default_server; + listen [::]:80 default_server; + + location / { + return 301 https://$host$request_uri; + } +} + +server { + + listen 443 ssl http2; + listen [::]:443 ssl http2; + + ssl_certificate /etc/certificates/epos-msl.pem; + ssl_certificate_key /etc/certificates/epos-msl.key; + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; # about 40000 sessions + ssl_session_tickets off; + + # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam + ssl_dhparam /etc/certificates/dhparams.pem; + + # intermediate configuration + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + # verify chain of trust of OCSP response using Root CA and Intermediate certs + ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt; + + client_max_body_size 100M; + + location / { + proxy_pass http://ckan:8080/; + proxy_set_header Host $host; + proxy_cache cache; + proxy_cache_bypass $cookie_auth_tkt; + proxy_no_cache $cookie_auth_tkt; + proxy_cache_valid 30m; + proxy_cache_key $host$scheme$proxy_host$request_uri; + # In emergency comment out line to force caching + # proxy_ignore_headers X-Accel-Expires Expires Cache-Control; + } +} diff --git a/docker/images/nginx/epos-msl.cnf b/docker/images/nginx/epos-msl.cnf new file mode 100644 index 0000000..6cd1532 --- /dev/null +++ b/docker/images/nginx/epos-msl.cnf @@ -0,0 +1,13 @@ +[ req ] +default_bits = 4096 +default_md = sha256 +prompt = no +encrypt_key = no +distinguished_name = dn + +[ dn ] +CN = "EPOS_MSL_HOST" +C = NL +O = Yoda +OU = ITS +L = Utrecht diff --git a/docker/images/nginx/nginx-entrypoint.sh b/docker/images/nginx/nginx-entrypoint.sh new file mode 100644 index 0000000..8043edd --- /dev/null +++ b/docker/images/nginx/nginx-entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +## Adapt URL in config +export EPOS_MSL_HOST="$EPOS_MSL_HOST" +perl -pi.bak -e '$epos_msl_host=$ENV{EPOS_MSL_HOST}; s/EPOS_MSL_HOST/$epos_msl_host/ge' /etc/certificates/epos-msl.cnf + +## Generate certificates if needed +cd /etc/certificates +if [ -f "epos-msl.pem" ] +then echo "Skipping certificate generation, because certificate files are already present." +else echo "Generating certificates for reverse proxy at https://$EPOS_MSL_HOST ..." + openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout epos-msl.key -out epos-msl.pem -config epos-msl.cnf + echo "Certificate generation complete." +fi + +if [ -f "dhparam.pem" ] +then echo "Skipping DHParam generation, because DHParam file is already present." +else echo "Generating DHParam configuration..." + openssl dhparam -dsaparam -out dhparams.pem 4096 + echo "DHParam generation complete." +fi + +## Run Nginx +nginx -g "daemon off;" diff --git a/docker/images/solr/Dockerfile b/docker/images/solr/Dockerfile new file mode 100644 index 0000000..e9028da --- /dev/null +++ b/docker/images/solr/Dockerfile @@ -0,0 +1,27 @@ +# Image for Solr configured for CKAN + +FROM solr:8.11 +MAINTAINER Yoda team + +ENV SOLR_CONFIG_DIR="/opt/solr/server/solr/configsets" + +# Create a CKAN configset based on the default one +USER root +RUN cp -R $SOLR_CONFIG_DIR/_default $SOLR_CONFIG_DIR/ckan + +# Download the EPOS-MSL core plugin (for the schema) +RUN cd /usr/lib && \ + apt update && \ + apt install -y git && \ + git clone https://github.com/UtrechtUniversity/msl_ckan_core.git + +# Change the CKAN configset to use the EPOS-MSL schema, and enable Query Elevation +USER root +ADD solrconfig.xml /opt/solr-8.11.4/server/solr/configsets/ckan/conf/solrconfig.xml +RUN ln -s /usr/lib/msl_ckan_core/ckanext/msl_ckan/config/solr/schema.xml $SOLR_CONFIG_DIR/ckan/conf/schema.xml && \ + perl -n -i.bak -e 'print unless /defaultSearchField/ or /solrQueryParser/' $SOLR_CONFIG_DIR/ckan/conf/schema.xml && \ + cp $SOLR_CONFIG_DIR/ckan/conf/schema.xml $SOLR_CONFIG_DIR/ckan/conf/managed-schema && \ + cp /opt/solr/server/solr/configsets/sample_techproducts_configs/conf/elevate.xml $SOLR_CONFIG_DIR/ckan/conf/elevate.xml + +USER solr +CMD ["sh", "-c", "solr-precreate ckan $SOLR_CONFIG_DIR/ckan"] diff --git a/docker/images/solr/build.sh b/docker/images/solr/build.sh new file mode 100755 index 0000000..957e4cd --- /dev/null +++ b/docker/images/solr/build.sh @@ -0,0 +1,2 @@ +#!/bin/sh +docker build -t epos-msl-solr:latest . diff --git a/docker/images/solr/solrconfig.xml b/docker/images/solr/solrconfig.xml new file mode 100644 index 0000000..43edf9b --- /dev/null +++ b/docker/images/solr/solrconfig.xml @@ -0,0 +1,1360 @@ + + + + + + + + 6.6.3 + + + + + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + AND + explicit + 10 + + + + + + + + + + + + + + + explicit + json + true + + + + + + + + explicit + + + + + + _text_ + + + + + + + true + ignored_ + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + + + tvComponent + + + + + + + + + + + + true + false + + + terms + + + + + + + + string + elevate.xml + + + + + + explicit + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSZ + yyyy-MM-dd'T'HH:mm:ss,SSSZ + yyyy-MM-dd'T'HH:mm:ss.SSS + yyyy-MM-dd'T'HH:mm:ss,SSS + yyyy-MM-dd'T'HH:mm:ssZ + yyyy-MM-dd'T'HH:mm:ss + yyyy-MM-dd'T'HH:mmZ + yyyy-MM-dd'T'HH:mm + yyyy-MM-dd HH:mm:ss.SSSZ + yyyy-MM-dd HH:mm:ss,SSSZ + yyyy-MM-dd HH:mm:ss.SSS + yyyy-MM-dd HH:mm:ss,SSS + yyyy-MM-dd HH:mm:ssZ + yyyy-MM-dd HH:mm:ss + yyyy-MM-dd HH:mmZ + yyyy-MM-dd HH:mm + yyyy-MM-dd + + + + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + ${velocity.template.base.dir:} + ${velocity.solr.resource.loader.enabled:true} + ${velocity.params.resource.loader.enabled:false} + + + + + 5 + + + + + + + + + + + + + + +