Merge pull request #1845 from nextcloud/release/v1.53.0

Release/v1.53.0

.github/workflows/build-docker.yml

@@ -154,7 +154,7 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Setup Selenium
-        run: pip install selenium==4.10.0
+        run: pip install selenium
 
       - name: Wait for container startup
         run: |
@@ -421,7 +421,7 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Setup Selenium
-        run: pip install selenium==4.10.0
+        run: pip install selenium
       - name: Wait for container startup
         run: |
           set -e

.github/workflows/build-lxd.yml

@@ -208,7 +208,7 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Setup Selenium
-        run: pip install selenium==4.10.0
+        run: pip install selenium
       - name: download LXD image from artifact store
         uses: actions/download-artifact@v3
         with:
@@ -331,7 +331,7 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Setup Selenium
-        run: pip install selenium==4.10.0
+        run: pip install selenium
       - name: download LXD image from artifact store
         uses: actions/download-artifact@v3
         with:

.github/workflows/build-sd-images.yml

@@ -173,7 +173,7 @@ jobs:
           set -x
           mv output/${ARTIFACT_FILE?} ncp.img
           sudo apt-get install -y systemd-container
-          sudo pip install selenium==4.10.0
+          sudo pip install selenium
           sudo rm -rf raspbian_root
           . ./build/buildlib.sh
           mount_raspbian "ncp.img"

.github/workflows/publish-image.yml

@@ -64,7 +64,7 @@ jobs:
           success=false
           for i in {1..5}
           do
-            body="$(hub release show -f "%b" "${VERSION}")"
+            body="$(gh release view --json body "${VERSION}" | jq -r '.body')"
             if ! [[ "$body" =~ .*'**Checksums:**'.* ]]
             then
             
@@ -80,7 +80,7 @@ jobs:
           \`\`\`"
           
             gh release edit "${VERSION?}" -n "$body"
-            if hub release show -f "%b" "${VERSION}" | grep "$checksum"
+            if gh release view --json body "${VERSION}" | jq -r '.body' | grep "$checksum"
             then
               success=true
               break

.github/workflows/vm-tests.yml

@@ -145,7 +145,8 @@ jobs:
           setup-ssh-port-forwarding "$SERVER_ADDRESS"
           
           echo "Run integration tests"
-          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
+          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg
+          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
     
             echo "Integration tests failed"
             echo "Here are the last lines of ncp-install.log:"
@@ -217,7 +218,7 @@ jobs:
           echo "Run integration tests"
           setup-ssh-port-forwarding "$SERVER_ADDRESS"
           
-          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
+          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
           
             echo "Integration tests failed"
             echo "Here are the last lines of ncp-install.log:"

bin/ncp-check-version

@@ -16,7 +16,9 @@ git clone -b "$BRANCH" --depth 20 -q --bare https://github.com/nextcloud/nextclo
 cd "$TEMPDIR" || exit 1
 VER=$( git describe --always --tags | grep -oP "v\d+\.\d+\.\d+" )
 
-canary="$(get_app_param ncp-community.sh CANARY)"
+canary="$(. /usr/local/etc/library.sh; find_app_param ncp-community.sh CANARY)"
+# TODO: Remove temporary canary override after staged rollouts test
+canary="no"
 
 [[ "$canary" != "yes" ]] && [[ -f "/usr/local/etc/instance.cfg" ]] && {
   cohorte_id="$(jq .cohorteId /usr/local/etc/instance.cfg)"

bin/ncp-provisioning.sh

@@ -3,7 +3,7 @@
 # this script runs at startup to provide an unique random passwords for each instance
 
 source /usr/local/etc/library.sh
-
+set -x
 ## redis provisioning
 
 CFG=/var/www/nextcloud/config/config.php
@@ -73,12 +73,13 @@ if needs_decrypt; then
 fi
 
 [[ -f /usr/local/etc/instance.cfg ]] || {
-  cohorte_id=$((1 + RANDOM % 100))
+  cohorte_id=$((RANDOM % 100))
   cat > /usr/local/etc/instance.cfg <<EOF
 {
-  "cohorteId": ${cohorte_id},
+  "cohorteId": ${cohorte_id}
 }
 EOF
+  cat /usr/local/etc/instance.cfg
 }
 
 

changelog.md

@@ -1,10 +1,10 @@
 # NextcloudPi Changelog
 
-## [v1.52.5](https://github.com/nextcloud/nextcloudpi/tree/v1.52.5) (2023-10-10) Staged Rollouts and NC 27.1.2
+## [v1.52.5](https://github.com/nextcloud/nextcloudpi/tree/v1.52.5) (2023-10-10) Staged Rollouts and NC 27.1.3
 
 ### Changes
 
-- Add support for Nextcloud 27.1.2
+- Add support for Nextcloud 27.1.3
 - Add support for staged rollouts
 
 

etc/ncp-config.d/nc-nextcloud.cfg

@@ -9,7 +9,7 @@
     {
       "id": "VER",
       "name": "Version",
-      "value": "27.1.2"
+      "value": "27.1.3"
     },
     {
       "id": "MAXFILESIZE",

etc/ncp.cfg

@@ -1,5 +1,5 @@
 {
-	"nextcloud_version": "27.1.2",
+	"nextcloud_version": "27.1.3",
 	"php_version": "8.1",
 	"release": "bullseye"
 }

generate_cohortes.sh

@@ -3,7 +3,7 @@
 #min=$((101-${1:-50}))
 all=({100..1})
 cohortes=()
-for i in ${all[@]:0:$1}
+for i in "${all[@]:0:${1?}}"
 do
   cohorte_id=$((RANDOM % i))
   while [[ " ${cohortes[*]} " =~ .*" ${cohorte_id} ".* ]]

install.sh

@@ -35,10 +35,12 @@ apt-get install --no-install-recommends -y git ca-certificates sudo lsb-release
 # get install code
 if [[ "${CODE_DIR}" == "" ]]; then
   echo "Getting build code..."
-  CODE_DIR="${TEMPDIR}"/nextcloudpi
-  git clone -b "${BRANCH}" https://github.com/nextcloud/nextcloudpi.git "${CODE_DIR}"
+  CODE_DIR_TMP="${TEMPDIR}"/nextcloudpi
+  git clone -b "${BRANCH}" https://github.com/nextcloud/nextcloudpi.git "${CODE_DIR_TMP}"
+  cd "$CODE_DIR_TMP"
+else
+  cd "${CODE_DIR}"
 fi
-cd "${CODE_DIR}"
 
 # install NCP
 echo -e "\nInstalling NextCloudPi..."

tests/activation_tests.py

@@ -163,13 +163,13 @@ def test_activation(IP, nc_port, admin_port, options, wait_timeout=120):
         sys.exit(2)
 
     arg_timeout = 120
-    options = Options()
+    options = webdriver.FirefoxOptions()
     for opt, arg in opts:
         if opt in ('-h', '--help'):
             usage()
             sys.exit(2)
         elif opt == '--no-gui':
-            options.headless = True
+            options.add_argument("-headless")
         elif opt in ('-t', '--timeout'):
             arg_timeout = int(arg)
         else:

tests/nextcloud_tests.py

@@ -148,7 +148,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver):
     test.report("password", "Wrong password" not in driver.page_source, msg="Failed to login with provided password")
 
     test.new("settings config")
-    wait = WebDriverWait(driver, 30)
+    wait = WebDriverWait(driver, 60)
     try:
         wait.until(VisibilityOfElementLocatedByAnyLocator([(By.CSS_SELECTOR, "#security-warning-state-ok"),
                                                            (By.CSS_SELECTOR, "#security-warning-state-warning"),
@@ -279,7 +279,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver):
         usage()
         sys.exit(2)
 
-    options = Options()
+    options = webdriver.FirefoxOptions()
     for opt, arg in opts:
         if opt in ('-h', '--help'):
             usage()
@@ -288,7 +288,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver):
             if os.path.exists(test_cfg):
                 os.unlink(test_cfg)
         elif opt == '--no-gui':
-            options.headless = True
+            options.add_argument("-headless")
         else:
             usage()
             sys.exit(2)

tests/system_tests.py

@@ -11,14 +11,17 @@
 
 More at https://ownyourbits.com
 """
+import json
+import subprocess
 
 pre_cmd = []
 
 import sys
 import getopt
 import os
 import signal
-from subprocess import run, getstatusoutput, PIPE
+from subprocess import run, getstatusoutput, PIPE, CompletedProcess
+from typing import Optional
 
 processes_must_be_running = [
         'apache2',
@@ -174,23 +177,95 @@ def signal_handler(sig, frame):
         sys.exit(0)
 
 
+class ProcessExecutionException(Exception):
+    pass
+
+
+def test_autoupdates():
+    def handle_error(r: CompletedProcess) -> CompletedProcess:
+        if r.returncode != 0:
+            print(f"{tc.red}error{tc.normal}\n{r.stdout.decode('utf-8') if r.stdout else ''}\n{r.stderr.decode('utf-8') if r.stderr else ''}"
+                  f" -- command failed: '{' '.join(r.args)}'")
+            raise ProcessExecutionException()
+        return CompletedProcess(r.args,
+                                r.returncode,
+                                r.stdout.decode('utf-8') if r.stdout else '',
+                                r.stderr.decode('utf-8') if r.stderr else '')
+
+    def set_cohorte_id(cohorte_id: int) -> CompletedProcess:
+        proc = subprocess.Popen(pre_cmd + ['cat', '/usr/local/etc/instance.cfg'], stdout=subprocess.PIPE, shell=False)
+        #handle_error(run(pre_cmd + ['cat', '/usr/local/etc/instance.cfg'], stdout=subprocess.STDOUT, stderr=subprocess.STDOUT))
+        #r = handle_error(run(pre_cmd + ['cat', '/usr/local/etc/instance.cfg'], stdout=PIPE, stderr=PIPE))
+        (out, err) = proc.communicate()
+        if proc.returncode != 0:
+            raise ProcessExecutionException()
+        try:
+            instance_cfg = json.loads(out)
+        except json.decoder.JSONDecodeError as e:
+            print(f"{tc.red}error{tc.normal} /usr/local/etc/instance.cfg could not be parsed, was: {out}\n{err}")
+            print(f"Command: '{' '.join(pre_cmd + ['cat', '/usr/local/etc/instance.cfg'])}'")
+            raise e
+
+        instance_cfg['cohorteId'] = cohorte_id
+        return handle_error(run(pre_cmd + ['bash', '-c', f'echo \'{json.dumps(instance_cfg)}\' > /usr/local/etc/instance.cfg'], stdout=PIPE, stderr=PIPE))
+
+    print(f"[updates] {tc.brown}staged rollouts{tc.normal}", end=' ')
+    try:
+        result = handle_error(run(pre_cmd + ['cat', '/usr/local/etc/ncp-version'], stdout=PIPE, stderr=PIPE))
+        if 'v99.99.99' in result.stdout:
+            print(f"{tc.yellow}skipped{tc.normal} (already updated to v99.99.99)")
+            return True
+        handle_error(run(pre_cmd + ['rm', '-f', '/var/run/.ncp-latest-version']))
+        handle_error(run(pre_cmd + ['sed', '-i', 's|BRANCH="master"|BRANCH="testing/staged-rollouts-1"|', '/usr/local/bin/ncp-check-version'], stdout=PIPE, stderr=PIPE))
+        set_cohorte_id(1)
+        result = run(pre_cmd + ['test', '-f', '/var/run/.ncp-latest-version'], stdout=PIPE, stderr=PIPE)
+        if result.returncode == 0:
+            result = handle_error(run(pre_cmd + ['cat', '/var/run/.ncp-latest-version'], stdout=PIPE, stderr=PIPE))
+            if 'v99.99.99' in result.stdout:
+                print(f"{tc.red}error{tc.normal} Auto update to v99.99.99 was unexpectedly not prevented by disabled cohorte id")
+                return False
+
+        set_cohorte_id(99)
+        handle_error(run(pre_cmd + ['/usr/local/bin/ncp-check-version'], stdout=PIPE, stderr=PIPE))
+        result = handle_error(run(pre_cmd + ['cat', '/var/run/.ncp-latest-version'], stdout=PIPE, stderr=PIPE))
+        if 'v99.99.99' not in result.stdout:
+            print(f"{tc.red}error{tc.normal} Expected latest detected version to be v99.99.99, was {result.stdout}")
+            return False
+
+        handle_error(run(pre_cmd + ['/usr/local/bin/ncp-test-updates']))
+        handle_error(run(pre_cmd + ['ncp-update', 'testing/staged-rollouts-1'], stdout=PIPE, stderr=PIPE))
+        result = handle_error(run(pre_cmd + ['cat', '/usr/local/etc/v99.99.99.success'], stdout=PIPE, stderr=PIPE))
+        if 'updated' not in result.stdout:
+            print(f"{tc.red}error{tc.normal} update to v99.99.99 did not succeed")
+            return False
+        print(f"{tc.green}ok{tc.normal}")
+
+    except ProcessExecutionException:
+        return False
+
+    return True
+
 if __name__ == "__main__":
 
     signal.signal(signal.SIGINT, signal_handler)
 
     # parse options
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'h', ['help', 'no-ping', 'non-interactive'])
+        opts, args = getopt.getopt(sys.argv[1:], 'h', ['help', 'no-ping', 'non-interactive', 'skip-update-test'])
     except getopt.GetoptError:
         usage()
         sys.exit(2)
 
     skip_ping = False
     interactive = True
+    skip_update_test = False
     for opt, arg in opts:
         if opt in ('-h', '--help'):
             usage()
             sys.exit(2)
+        elif opt == '--skip-update-test':
+            print("Skipping update test")
+            skip_update_test = True
         elif opt == '--no-ping':
             skip_ping = True
         elif opt == '--non-interactive':
@@ -256,7 +331,7 @@ def signal_handler(sig, frame):
                tc.yellow + ssh_cmd + tc.normal + "...")
         binaries_must_be_installed = binaries_must_be_installed + binaries_no_docker
         pre_cmd = ['ssh', '-o UserKnownHostsFile=/dev/null' , '-o PasswordAuthentication=no',
-                '-o StrictHostKeyChecking=no', '-o ConnectTimeout=1', ssh_cmd[4:]]
+                   '-o StrictHostKeyChecking=no', '-o ConnectTimeout=10', ssh_cmd[4:]]
 
         if not skip_ping:
             at_char = ssh_cmd.index('@')
@@ -282,8 +357,9 @@ def signal_handler(sig, frame):
     files1_result  = check_files_exist(files_must_exist)
     files2_result  = check_files_dont_exist(files_must_not_exist)
     notify_push_result = check_notify_push()
+    update_test_result = True if skip_update_test else test_autoupdates()
 
-    if running_result and install_result and files1_result and files2_result and notify_push_result:
+    if running_result and install_result and files1_result and files2_result and notify_push_result and update_test_result:
         sys.exit(0)
     else:
         sys.exit(1)

updates/1.52.5.sh → updates/1.53.0sh

@@ -4,7 +4,7 @@
   cohorte_id=$((RANDOM % 100))
   cat > /usr/local/etc/instance.cfg <<EOF
 {
-  "cohorteId": ${cohorte_id},
+  "cohorteId": ${cohorte_id}
 }
 EOF
 }