Update docs/laptop

docs/laptop/README.md

@@ -52,15 +52,20 @@ docker network create paddles
 Start postgres containers in order to use paddles:
 
 ```bash
-mkdir $HOME/.teuthology/postgres
+mkdir -p $HOME/.teuthology/postgres
 docker run -d -p 5432:5432 --network paddles --name paddles-postgres \
     -e POSTGRES_PASSWORD=secret \
     -e POSTGRES_USER=paddles \
     -e POSTGRES_DB=paddles \
     -e PGDATA=/var/lib/postgresql/data/pgdata \
+    -e PADDLES_JOB_LOG_HREF_TEMPL='http://localhost:8888/{run_name}/{job_id}/teuthology.log' \
     -v $HOME/.teuthology/postgres:/var/lib/postgresql/data postgres
 ```
 
+NOTE. When running container on MacOS X using podman postgres may experience
+troubles with volume directory binds because of podman machine,  thus use regular
+volumes like `-v paddlesdb:/var/lib/postgresql/data`.
+
 ### Run paddles
 
 Checkout paddles and build the image:
@@ -72,7 +77,7 @@ cd ~/paddles && docker build . --file Dockerfile --tag paddles
 Run the container with previously created network:
 
 ```bash
-docker run -d --network paddles --name api -p 80:8080 \
+docker run -d --network paddles --name api -p 8080:8080 \
 	-e PADDLES_SERVER_HOST=0.0.0.0 \
 	-e PADDLES_SQLALCHEMY_URL=postgresql+psycopg2://paddles:secret@paddles-postgres/paddles \
 	paddles

docs/siteconfig.rst

@@ -255,6 +255,17 @@ Here is a sample configuration with many of the options set and documented::
       endpoint: http://head.ses.suse.de:5000/
       machine_types: ['type1', 'type2', 'type3']
 
+    # Define a list of ssh tunnels for a various group of test nodes.
+    # Notice: provided domain names for the nodes must be resolvable
+    # in your network and jump host (bastion) must be accessible.
+    tunnel:
+      - hosts: ['example1.domain', 'example2.domain', 'example3.domain']
+        bastion:
+          host: ssh_host_name         # must be resolvable and reachable
+          user: ssh_user_name         # (optional)
+          port: ssh_port              # (optional)
+          identity: ~/.ssh/id_ed25519 # (optional)
+
     # Do not allow more than that many jobs in a single run by default.
     # To disable this check use 0.
     job_threshold: 500

setup.cfg

@@ -56,6 +56,7 @@ install_requires =
     python-dateutil
     requests>2.13.0
     sentry-sdk
+    sshtunnel
     types-psutil
     urllib3>=1.25.4,<1.27  # For botocore
 scripts =

teuthology/lock/cli.py

@@ -178,12 +178,14 @@ def main(ctx):
                 # Update keys last
                 updatekeys_machines = list()
             else:
-                machines_to_update.append(machine)
                 ops.update_nodes([machine], True)
-                teuthology.provision.create_if_vm(
+                created = teuthology.provision.create_if_vm(
                     ctx,
                     misc.canonicalize_hostname(machine),
                 )
+                # do not try to update inventory if failed to create vm
+                if created:
+                    machines_to_update.append(machine)
         with teuthology.parallel.parallel() as p:
             ops.update_nodes(reimage_machines, True)
             for machine in reimage_machines:

teuthology/lock/ops.py

@@ -445,7 +445,6 @@ def block_and_lock_machines(ctx, total_requested, machine_type, reimage=True, tr
                     loopcount += 1
                     time.sleep(10)
                     keys_dict = misc.ssh_keyscan(vmlist)
-                    log.info('virtual machine is still unavailable')
                     if loopcount == 40:
                         loopcount = 0
                         log.info('virtual machine(s) still not up, ' +

teuthology/misc.py

@@ -1112,6 +1112,11 @@ def _ssh_keyscan(hostname):
     :returns: The host key
     """
     args = ['ssh-keyscan', '-T', '1', hostname]
+    if config.tunnel:
+        for tunnel in config.tunnel:
+            if hostname in tunnel.get('hosts'):
+                bastion = tunnel.get('bastion')
+                args = ['ssh', bastion.get('host')] + args
     p = subprocess.Popen(
         args=args,
         stdout=subprocess.PIPE,

teuthology/orchestra/connection.py

@@ -8,6 +8,7 @@
 from teuthology.config import config
 from teuthology.contextutil import safe_while
 from paramiko.hostkeys import HostKeyEntry
+import sshtunnel
 
 log = logging.getLogger(__name__)
 
@@ -50,7 +51,20 @@ def connect(user_at_host, host_key=None, keep_alive=False, timeout=60,
     :param retry:       Whether or not to retry failed connection attempts
                         (eventually giving up if none succeed). Default is True
     :param key_filename:  Optionally override which private key to use.
+
     :return: ssh connection.
+
+    The connection is going to be established via tunnel if corresponding options
+    are provided in teuthology configuration file. For example:
+
+    tunnel:
+      - hosts: ['hostname1.domain', 'hostname2.domain']
+        bastion:
+          host: ssh_host_name
+          user: ssh_user_name
+          port: 22
+          identity: ~/.ssh/id_ed25519
+
     """
     user, host = split_user(user_at_host)
     if _SSHClient is None:
@@ -79,6 +93,29 @@ def connect(user_at_host, host_key=None, keep_alive=False, timeout=60,
         timeout=timeout
     )
 
+    if config.tunnel:
+        for tunnel in config.tunnel:
+            if host in tunnel.get('hosts'):
+                bastion = tunnel.get('bastion')
+                if not bastion:
+                    log.error("The 'tunnel' config must include 'bastion' entry")
+                    continue
+                bastion_host = bastion.get('host')
+                server = sshtunnel.SSHTunnelForwarder(
+                    bastion_host,
+                    ssh_username=bastion.get('user', None),
+                    ssh_password=bastion.get('word', None),
+                    ssh_pkey=bastion.get('identity'),
+                    remote_bind_address=(host, 22))
+                log.info(f'Starting tunnel to {bastion_host} for host {host}')
+                server.start()
+                local_port = server.local_bind_port
+                log.debug(f"Local port for host {host} is {local_port}")
+                connect_args['hostname'] = '127.0.0.1'
+                connect_args['port'] = local_port
+                ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+                break
+
     key_filename = key_filename or config.ssh_key
     ssh_config_path = config.ssh_config_path or "~/.ssh/config"
     ssh_config_path = os.path.expanduser(ssh_config_path)

teuthology/provision/downburst.py

@@ -107,8 +107,9 @@ def create(self):
                         self.destroy()
                     else:
                         success = False
-                        log.info("Downburst failed on %s: %s" % (
-                            self.name, stderr.strip()))
+                        log.error("Downburst failed on %s" % self.name)
+                        for i in stderr.split('\n'):
+                            log.error(f">>> {i}")
                         break
             return success
 

teuthology/task/ansible.py

@@ -259,12 +259,19 @@ def generate_inventory(self):
         we're using an existing file.
         """
         hosts = self.cluster.remotes.keys()
-        hostnames = [remote.hostname for remote in hosts]
-        hostnames.sort()
+        hostnames = []
+        for remote in hosts:
+            if remote.ssh:
+                host, port = remote.ssh.get_transport().getpeername()
+                i = f"{remote.hostname} ansible_host={host} ansible_port={port} ansible_ssh_common_args='-o StrictHostKeyChecking=no'"
+            else:
+                i = remote.hostname
+            hostnames.append(i)
         inventory = []
         if self.inventory_group:
             inventory.append('[{0}]'.format(self.inventory_group))
-        inventory.extend(hostnames + [''])
+
+        inventory.extend(sorted(hostnames) + [''])
         hosts_str = '\n'.join(inventory)
         self.inventory = self._write_inventory_files(hosts_str)
         self.generated_inventory = True

teuthology/task/kernel.py

@@ -861,8 +861,9 @@ def install_kernel(remote, role_config, path=None, version=None):
     if package_type == 'deb':
         newversion = get_latest_image_version_deb(remote, dist_release, role_config)
         if 'ubuntu' in dist_release:
-            grub2conf = teuthology.get_file(remote,
-                '/boot/grub/grub.cfg', sudo=True).decode()
+            #grub2conf = teuthology.get_file(remote,
+            #    '/boot/grub/grub.cfg', sudo=True).decode()
+            grub2conf = remote.read_file('/boot/grub/grub.cfg', sudo=True).decode()
             submenu = ''
             menuentry = ''
             for line in grub2conf.split('\n'):
@@ -882,9 +883,10 @@ def install_kernel(remote, role_config, path=None, version=None):
                 grubvalue = submenu + '>' + menuentry
             else:
                 grubvalue = menuentry
-            grubfile = 'cat <<EOF\nset default="' + grubvalue + '"\nEOF'
-            teuthology.delete_file(remote, '/etc/grub.d/01_ceph_kernel', sudo=True, force=True)
-            teuthology.sudo_write_file(remote, '/etc/grub.d/01_ceph_kernel', StringIO(grubfile), '755')
+            path = '/etc/grub.d/01_ceph_kernel'
+            data = f"cat <<EOF\nset default=\"{grubvalue}\"\nEOF"
+            remote.sh(f"sudo rm -rf {path}")
+            remote.write_file(path, data, sudo=True, mode='755')
             log.info('Distro Kernel Version: {version}'.format(version=newversion))
             remote.run(args=['sudo', 'update-grub'])
             remote.run(args=['sudo', 'shutdown', '-r', 'now'], wait=False )
@@ -1077,19 +1079,17 @@ def get_latest_image_version_deb(remote, ostype, role_config):
     """
     remote.run(args=['sudo', 'apt-get', 'clean'])
     remote.run(args=['sudo', 'apt-get', 'update'])
-    output = StringIO()
     newest = ''
     # Depend of virtual package has uname -r output in package name. Grab that.
     # Note that a dependency list may have multiple comma-separated entries,
     # but also each entry may be an alternative (pkg1 | pkg2)
     if 'debian' in ostype:
         args=['sudo', 'apt-get', '-y', 'install', 'linux-image-amd64']
         install_dep_packages(remote, args)
-        remote.run(args=['dpkg', '-s', 'linux-image-amd64'], stdout=output)
-        for line in output.getvalue().split('\n'):
+        status = remote.run("dpkg -s linux-image-amd64")
+        for line in status.split('\n'):
             if 'Depends:' in line:
                 newest = line.split('linux-image-')[1]
-                output.close()
                 return newest
     # Ubuntu is a depend in a depend.
     if 'ubuntu' in ostype:
@@ -1100,10 +1100,8 @@ def get_latest_image_version_deb(remote, ostype, role_config):
         args=['sudo', 'DEBIAN_FRONTEND=noninteractive',
               'apt-get', '-y', 'install', name]
         install_dep_packages(remote, args)
-        remote.run(args=['dpkg', '-s', name],
-                   stdout=output)
-
-        for line in output.getvalue().split('\n'):
+        status = remote.sh(f"dpkg -s {name}")
+        for line in status.split('\n'):
             if 'Depends:' in line:
                 newest = line.split('linux-image-')[1]
                 if ',' in newest:
@@ -1112,7 +1110,6 @@ def get_latest_image_version_deb(remote, ostype, role_config):
                     # not strictly correct, as any of the |-joined
                     # packages may satisfy the dependency
                     newest = newest.split('|')[0].strip()
-    output.close()
     return newest