Release AM 1.15.0 + SS 0.21.0

Co-authored-by: Douglas Cerna (Soy Douglas) <[email protected]>

.Jenkinsci

@@ -43,7 +43,7 @@ node {
   }
 
   stage('Build rpm packages') {
-    build job: 'am-packbuild/rpm-jenkinsci', parameters: [
+    build job: 'am-packbuild/rpm-EL9-jenkinsci', parameters: [
       string(name: 'PACKAGE', value: "${PACKAGE}"),
       string(name: 'BRANCH', value: 'qa/1.x'),
       string(name: 'GPG_ID', value: '0F4A4D31'),
@@ -52,8 +52,8 @@ node {
       string(name: 'REPOSITORY', value: "${REPOSITORY}")]
   }
 
-  stage('Build bionic packages') {
-    build job: 'am-packbuild/bionic-jenkinsci', parameters: [
+  stage('Build deb packages') {
+    build job: 'am-packbuild/jammy-jenkinsci', parameters: [
       string(name: 'PACKAGE', value: "${PACKAGE}"),
       string(name: 'BRANCH', value: 'qa/1.x'),
       string(name: 'GPG_ID', value: '0F4A4D31'),

.Jenkinsci-ss

@@ -43,7 +43,7 @@ node {
   }
 
   stage('Build rpm packages') {
-    build job: 'am-packbuild/rpm-jenkinsci', parameters: [
+    build job: 'am-packbuild/rpm-EL9-jenkinsci', parameters: [
       string(name: 'PACKAGE', value: "${PACKAGE}"),
       string(name: 'BRANCH', value: 'qa/0.x'),
       string(name: 'GPG_ID', value: '0F4A4D31'),
@@ -52,8 +52,8 @@ node {
       string(name: 'REPOSITORY', value: "${REPOSITORY}")]
   }
 
-  stage('Build bionic packages') {
-    build job: 'am-packbuild/bionic-jenkinsci', parameters: [
+  stage('Build deb packages') {
+    build job: 'am-packbuild/jammy-jenkinsci', parameters: [
       string(name: 'PACKAGE', value: "${PACKAGE}"),
       string(name: 'BRANCH', value: 'qa/0.x'),
       string(name: 'GPG_ID', value: '0F4A4D31'),

README.md

@@ -4,9 +4,9 @@ You need Docker installed and running.
 
 CentOS 7 packages:
 
-    make -C rpm
+    make -C rpm-EL7
 
-This also builds a local repository that you can use later from `rpm-testing`.
+This also builds a local repository that you can use later from `rpm-EL7-testing`.
 
 We don't have a single target yet for Debian packages but you can build them
 individually, for example:
@@ -69,7 +69,7 @@ managing Ubuntu repositories using reprepro can be found
 
 ### Test package
 
-CentOS 7 packages: see the [./rpm-testing](rpm-testing) directory for more
+CentOS 7 packages: see the [./rpm-EL7-testing](rpm-EL7-testing) directory for more
 details.
 
 Ubuntu packages: this is work in progress (see #127).

deb-testing/README.md

@@ -0,0 +1,20 @@
+This Vagrant environment is based on the official Ubuntu 22.04 Vagrant box
+`ubuntu/jammy64`.
+
+### Instructions
+
+Provision the Vagrant box using our official repository:
+
+    $ vagrant up
+
+Alternatively, provision the box using the local repository (`../debs/jammy`),
+which needs to be previously built:
+
+    # Build the packages and the local repo. Then create the box.
+    $ make -C ../debs/jammy
+    $ LOCAL_REPOSITORY="yes" vagrant up
+
+Once is up you should be able to access to the web interfaces:
+
+    - Access to Dashboard: http://192.168.33.2
+    - Access to Storage Service: http://192.168.33.2:8000

deb-testing/Vagrantfile

@@ -0,0 +1,19 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure(2) do |config|
+  config.vm.box = "ubuntu/jammy64"
+  config.vm.network "private_network", ip: "192.168.33.2"
+  config.vm.synced_folder ".", "/vagrant"
+  config.vm.synced_folder "../", "/am-packbuild"
+  config.vm.provider "virtualbox" do |vb|
+    vb.memory = "4096"
+    vb.cpus = "2"
+  end
+  config.vm.provision "shell",
+    inline: "/am-packbuild/deb-testing/install.sh",
+    env: {
+      "LOCAL_REPOSITORY" => ENV.fetch('LOCAL_REPOSITORY', 'no'),
+      "SEARCH_ENABLED" => ENV.fetch('SEARCH_ENABLED', 'no'),
+    }
+end

deb-testing/install.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -x
+
+
+function get_env_boolean() {
+    local name="$1"
+    local default="$2"
+    local ret="${default}"
+    if [ "${default}" == "true" ]; then
+        if [ "${!name}" == "no" ] || [ "${!name}" == "false" ] || [ "${!name}" == "0" ]; then
+            ret="false"
+        fi
+    fi
+    if [ "${default}" == "false" ]; then
+        if [ "${!name}" == "yes" ] || [ "${!name}" == "true" ] || [ "${!name}" == "1" ]; then
+            ret="true"
+        fi
+    fi
+    echo -n "${ret}"
+}
+
+search_enabled=$(get_env_boolean "SEARCH_ENABLED" "true")
+local_repository=$(get_env_boolean "LOCAL_REPOSITORY" "false")
+
+echo "~~~~~~~~ DEBUG ~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+while read -r line; do echo "$line=${!line}"; done < <(compgen -v | grep -v '[^[:lower:]_]' | grep -v '^_$')
+echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+
+export DEBIAN_FRONTEND=noninteractive
+
+sudo wget -O - https://packages.archivematica.org/1.15.x/key.asc | sudo apt-key add -
+sudo sh -c 'echo "deb [arch=amd64] http://packages.archivematica.org/1.15.x/ubuntu-externals jammy main" >> /etc/apt/sources.list'
+
+if [ "${local_repository}" == "true" ] ; then
+    sudo -u root bash -c 'cat << EOF > /etc/apt/sources.list.d/archivematica.list
+deb file:/am-packbuild/debs/jammy/_deb_repository ./
+EOF'
+else
+    sudo sh -c 'echo "deb [arch=amd64] http://packages.archivematica.org/1.15.x/ubuntu jammy main" >> /etc/apt/sources.list'
+fi
+
+sudo apt-get -o Acquire::AllowInsecureRepositories=true update
+sudo apt-get -y upgrade
+sudo apt-get install -y htop ntp apt-transport-https unzip openjdk-8-jre-headless mysql-server python3-mysqldb
+
+if [ "${search_enabled}" == "true" ] ; then
+    wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
+    echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-6.x.list
+    sudo apt-get -o Acquire::AllowInsecureRepositories=true update
+    sudo apt-get install -y elasticsearch
+    sudo systemctl daemon-reload
+    sudo service elasticsearch restart
+    sudo systemctl enable elasticsearch
+fi
+
+sudo -H -u root mysql -hlocalhost -uroot -e "DROP DATABASE IF EXISTS SS; CREATE DATABASE SS CHARACTER SET utf8 COLLATE utf8_unicode_ci;"
+sudo -H -u root mysql -hlocalhost -uroot -e "CREATE USER 'archivematica'@'localhost' IDENTIFIED BY 'demo';"
+sudo -H -u root mysql -hlocalhost -uroot -e "GRANT ALL ON SS.* TO 'archivematica'@'localhost';"
+
+sudo apt-get install -y --allow-unauthenticated archivematica-storage-service
+
+sudo rm -f /etc/nginx/sites-enabled/default
+sudo ln -sf /etc/nginx/sites-available/storage /etc/nginx/sites-enabled/storage
+
+sudo -H -u root mysql -hlocalhost -uroot -e "DROP DATABASE IF EXISTS MCP; CREATE DATABASE MCP CHARACTER SET utf8 COLLATE utf8_unicode_ci;"
+sudo -H -u root mysql -hlocalhost -uroot -e "GRANT ALL ON MCP.* TO 'archivematica'@'localhost';"
+
+echo archivematica-mcp-server archivematica-mcp-server/dbconfig-install boolean false | sudo debconf-set-selections
+echo postfix postfix/main_mailer_type select No configuration | sudo debconf-set-selections
+
+sudo apt-get install -y --allow-unauthenticated archivematica-mcp-server
+sudo apt-get install -y --allow-unauthenticated archivematica-dashboard
+sudo apt-get install -y --allow-unauthenticated archivematica-mcp-client
+
+if [ "${search_enabled}" != "true" ] ; then
+    sudo sh -c 'echo "ARCHIVEMATICA_DASHBOARD_DASHBOARD_SEARCH_ENABLED=false" >> /etc/default/archivematica-dashboard'
+    sudo sh -c 'echo "ARCHIVEMATICA_MCPSERVER_MCPSERVER_SEARCH_ENABLED=false" >> /etc/default/archivematica-mcp-server'
+    sudo sh -c 'echo "ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_SEARCH_ENABLED=false" >> /etc/default/archivematica-mcp-client'
+fi
+
+sudo ln -sf /etc/nginx/sites-available/dashboard.conf /etc/nginx/sites-enabled/dashboard.conf
+
+sudo service clamav-freshclam restart
+sleep 120s
+sudo service clamav-daemon start
+sudo service gearman-job-server restart
+sudo service archivematica-mcp-server start
+sudo service archivematica-mcp-client restart
+sudo service archivematica-storage-service start
+sudo service archivematica-dashboard restart
+sudo service nginx restart
+sudo systemctl enable fits-nailgun
+sudo service fits-nailgun start

debs/jammy/.Jenkinsci

@@ -5,7 +5,7 @@ pipeline {
   string(defaultValue: 'qa/1.x', description: '', name: 'BRANCH')
   string(defaultValue: '', description: '', name: 'GIT_REPO')
   string(defaultValue: '0F4A4D31', description: 'For production packages, use production key', name: 'GPG_ID')
-  string(defaultValue: '1.13.0', description: '', name: 'VERSION')
+  string(defaultValue: '1.15.0', description: '', name: 'VERSION')
   string(defaultValue: '-beta1', description: '', name: 'RELEASE')
   string(defaultValue: 'qa/1.x', description: '', name: 'PACKBUILD_BRANCH')
   string(defaultValue: 'jenkinsci', description: '', name: 'REPOSITORY')

debs/jammy/Makefile

@@ -0,0 +1,18 @@
+DEB_REPOSITORY = "$(shell pwd)/_deb_repository"
+
+all: build createrepo
+
+build:
+	make -C ./archivematica-storage-service
+	make -C ./archivematica
+
+createrepo: cleanrepo
+	mkdir -p $(DEB_REPOSITORY) || true
+	find ./archivematica-storage-service ./archivematica -name "*.deb" | grep -v src | xargs -IF cp -f F $(DEB_REPOSITORY)
+	docker run --rm --volume "$(DEB_REPOSITORY):/deb-repository" ubuntu:22.04 bash -c "apt-get update && apt-get install -y dpkg-dev && cd /deb-repository && dpkg-scanpackages . /dev/null | gzip -9c > Packages.gz"
+
+clean: cleanrepo
+	find . -name "*.deb" -delete
+
+cleanrepo:
+	rm -rf $(DEB_REPOSITORY)

debs/jammy/archivematica-storage-service/.dockerignore

@@ -0,0 +1,2 @@
+/src
+/repo

debs/jammy/archivematica-storage-service/Dockerfile

@@ -0,0 +1,24 @@
+FROM ubuntu:jammy
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update \
+	&& apt-get install -y \
+		apt-transport-https \
+		build-essential \
+		debhelper \
+		devscripts \
+		dh-virtualenv \
+		dpkg-dev \
+		equivs \
+		git \
+		libffi-dev \
+		libldap2-dev \
+		libmysqlclient-dev \
+		libsasl2-dev \
+		libssl-dev \
+		libxslt-dev \
+		pkg-config \
+		python3-dev \
+        software-properties-common \
+	&& rm -rf /var/lib/apt/lists/*

debs/jammy/archivematica-storage-service/Makefile

@@ -0,0 +1,89 @@
+NAME          = am-packbuild
+PACKAGE       = archivematica-storage-service
+DEB_TOPDIR    = "/debbuild"
+DOCKER_VOLUME = "/src"
+DOCKER_IMAGE  = "debbuild-$(NAME)-$(PACKAGE)-$(VERSION)"
+GPG_ID        ?= 0F4A4D31
+BRANCH        ?= qa/0.x
+VERSION       ?= 0.21.0
+RELEASE       ?= -1
+GIT_REPO      = "https://github.com/artefactual/archivematica-storage-service"
+
+.DEFAULT_GOAL := build
+
+.PHONY: build-docker-image
+build-docker-image:
+	@echo "==> Building Docker image with build environment."
+	@docker build --rm --tag "$(DOCKER_IMAGE)" .
+
+.PHONY: build
+build: build-docker-image
+	@echo "==> Building deb."
+	@docker run \
+		-e BRANCH=$(BRANCH) \
+		-e VERSION=$(VERSION) \
+		-e RELEASE=$(RELEASE) \
+		-e PACKAGE=$(PACKAGE) \
+		-e GPG_ID=$(GPG_ID) \
+		-e GPG_KEY \
+		-e GIT_REPO="$(GIT_REPO)" \
+		--rm \
+		--volume "$(shell cd ../../ && pwd):$(DEB_TOPDIR)/$(NAME)" \
+		--volume "$(shell pwd):$(DOCKER_VOLUME)" \
+		$(DOCKER_IMAGE) \
+			make -C $(DOCKER_VOLUME) deb-build
+
+.PHONY: dev
+dev: build-docker-image
+	@echo "==> Building deb."
+	@docker run -i -t \
+		-e BRANCH=$(BRANCH) \
+		-e VERSION=$(VERSION) \
+		-e RELEASE=$(RELEASE) \
+		-e PACKAGE=$(PACKAGE) \
+		-e GPG_ID=$(GPG_ID) \
+		-e GPG_KEY \
+		-e GIT_REPO="$(GIT_REPO)" \
+		--rm \
+		--volume "$(shell cd ../../ && pwd):$(DEB_TOPDIR)/$(NAME)" \
+		--volume "$(shell pwd):$(DOCKER_VOLUME)" \
+		$(DOCKER_IMAGE) \
+			/bin/bash
+
+.PHONY: deb-build
+deb-build: deb-clean git-clone import-gpg-key
+	@cd /debbuild/$(NAME)
+	./build.sh
+	mkdir -p repo/
+	cp -rf src/archivematica-*.* repo
+	cd repo && dpkg-scanpackages . | gzip > Packages.gz
+
+.PHONY: import-gpg-key
+import-gpg-key:
+	@if [ -f "$(DOCKER_VOLUME)/GPG-KEY" ]; then gpg --import $(DOCKER_VOLUME)/GPG-KEY; fi
+	@if [ x"$$GPG_KEY" != x ]; then echo "$$GPG_KEY" | gpg --import - ; fi
+
+.PHONY: git-clone
+git-clone:
+	@git clone \
+		--quiet \
+		--branch $(BRANCH) \
+		--depth 1 \
+		--single-branch \
+			$(GIT_REPO) src/$(PACKAGE)
+
+.PHONY: deb-clean
+deb-clean:
+	@echo "==> Cleaning up previous builds."
+	@rm -rf $(DOCKER_VOLUME)/repo/
+	@rm -rf $(DOCKER_VOLUME)/src/
+
+.PHONY: cleanup
+cleanup: build-docker-image
+	@echo "==> Remove artifacts created as root."
+	@docker run -i \
+		--rm \
+		--volume "$(shell cd ../../ && pwd):$(DEB_TOPDIR)/$(NAME)" \
+		--volume "$(shell pwd):$(DOCKER_VOLUME)" \
+		$(DOCKER_IMAGE) \
+		rm -rf /src/repo/ /src/src/

debs/jammy/archivematica-storage-service/build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -euxo
+
+BASE="$(pwd)"
+SOURCE=${BASE}/src/archivematica-storage-service/
+export DEBFULLNAME="Artefactual Systems"
+export DEBEMAIL="[email protected]"
+export DEB_BUILD_OPTIONS="noddebs"
+
+cd $SOURCE
+COMMIT=$(git rev-parse HEAD)
+cp -rf ${BASE}/debian-storage-service debian
+yes | mk-build-deps -i debian/control
+dch -v 1:${VERSION}${RELEASE}~22.04 commit: $(echo $COMMIT)
+dch -v 1:${VERSION}${RELEASE}~22.04 checkout: $(echo $BRANCH) 
+dch -r --distribution jammy --urgency high ignored		
+dpkg-buildpackage -us -uc
+cd $SOURCE
+

debs/jammy/archivematica-storage-service/changelog.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+export DEBFULLNAME="Artefactual Systems"
+export DEBEMAIL="[email protected]"
+BRANCH="$(git branch | cut -d\  -f2-)"
+COMMIT=$(git rev-parse HEAD)
+
+dch -v 1:${VERSION}${RELEASE} commit: $(echo $COMMIT)
+dch -v 1:${VERSION}${RELEASE} checkout: $(echo $BRANCH) 
+dch -r --distribution jammy --urgency high ignored

debs/jammy/archivematica-storage-service/debian-storage-service/archivematica-storage-service.default

@@ -0,0 +1,15 @@
+PYTHONPATH=/usr/lib/archivematica/storage-service
+LANG="en_US.UTF-8"
+LC_ALL="en_US.UTF-8"
+LC_LANG="en_US.UTF-8"
+
+SS_DB_URL=mysql://archivematica:demo@localhost:3306/SS
+DJANGO_ALLOWED_HOSTS=*
+SS_DB_PASSWORD=
+SS_DB_USER=
+SS_DB_HOST=
+DJANGO_SETTINGS_MODULE=storage_service.settings.production
+DJANGO_SECRET_KEY=CHANGE_ME_WITH_A_SECRET_KEY
+SS_GUNICORN_BIND=127.0.0.1:8001
+
+REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt