Merge pull request #7 from RAKWireless/release-0.8.0

Release 0.8.0

CHANGELOG.txt

@@ -4,6 +4,24 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.0] - 2023-02-09
+
+### Changed
+- Based on upstream 2023-12-11-raspios-bookworm with kernel 6.1
+- OLED script as service
+- Refactor stage2-rak folders
+
+### Added
+- Added firstboot.d folder with scripts to execute on first boot
+- Support for Raspberry Pi 5
+
+### Fixed
+- Remove loopback interface when checking for connected networks in create-ap
+
+### Removed
+- Removed RAKUID variable and utility
+- Remove most python modules from default installation
+
 ## [0.7.1] - 2023-07-26
 
 ### Fixed

build-docker.sh

@@ -1,13 +1,20 @@
-#!/bin/bash -eu
+#!/usr/bin/env bash
+# Note: Avoid usage of arrays as MacOS users have an older version of bash (v3.x) which does not supports arrays
+set -eu
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)"
 
 BUILD_OPTS="$*"
 
-DOCKER="docker"
+# Allow user to override docker command
+DOCKER=${DOCKER:-docker}
 
-if ! ${DOCKER} ps >/dev/null 2>&1; then
-	DOCKER="sudo docker"
+# Ensure that default docker command is not set up in rootless mode
+if \
+  ! ${DOCKER} ps    >/dev/null 2>&1 || \
+    ${DOCKER} info 2>/dev/null | grep -q rootless \
+; then
+	DOCKER="sudo ${DOCKER}"
 fi
 if ! ${DOCKER} ps >/dev/null; then
 	echo "error connecting to docker:"
@@ -78,50 +85,89 @@ BUILD_OPTS="$(echo "${BUILD_OPTS:-}" | sed -E 's@\-c\s?([^ ]+)@-c /config@')"
 ${DOCKER} build --build-arg BASE_IMAGE=debian:bullseye -t pi-gen "${DIR}"
 
 if [ "${CONTAINER_EXISTS}" != "" ]; then
-	trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM
-	time ${DOCKER} run --rm --privileged \
-		--cap-add=ALL \
-		-v $(pwd):/pi-gen \
-		-v /dev:/dev \
-		-v /lib/modules:/lib/modules \
-		${PIGEN_DOCKER_OPTS} \
-		--volume "${CONFIG_FILE}":/config:ro \
-		-e "GIT_HASH=${GIT_HASH}" \
-		--volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \
-		pi-gen \
-		bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static &&
-	# binfmt_misc is sometimes not mounted with debian bullseye image
-	(mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc || true) &&
-	cd /pi-gen; ./build.sh ${BUILD_OPTS} &&
-	rsync -av work/*/build.log deploy/" &
-	wait "$!"
+  DOCKER_CMDLINE_NAME="${CONTAINER_NAME}_cont"
+  DOCKER_CMDLINE_PRE="--rm"
+  DOCKER_CMDLINE_POST="--volumes-from=${CONTAINER_NAME}"
 else
-	trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM
-	time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \
-		--cap-add=ALL \
-		-v $(pwd):/pi-gen \
-		-v /dev:/dev \
-		-v /lib/modules:/lib/modules \
-		${PIGEN_DOCKER_OPTS} \
-		--volume "${CONFIG_FILE}":/config:ro \
-		-e "GIT_HASH=${GIT_HASH}" \
-		pi-gen \
-		bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static &&
-	# binfmt_misc is sometimes not mounted with debian bullseye image
-	(mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc || true) &&
-	cd /pi-gen; ./build.sh ${BUILD_OPTS} &&
-	rsync -av work/*/build.log deploy/" &
-	wait "$!"
+  DOCKER_CMDLINE_NAME="${CONTAINER_NAME}"
+  DOCKER_CMDLINE_PRE=""
+  DOCKER_CMDLINE_POST=""
 fi
 
-rm -rf build
-${DOCKER} cp -q ${CONTAINER_NAME}:/pi-gen/deploy build
-${DOCKER} cp -q ${CONTAINER_NAME}:/pi-gen/stage2-rak/02-kernel/files/${ARCH}.kernel.zip build/${ARCH}.kernel.zip
-ls -lah build
+# Check if binfmt_misc is required
+binfmt_misc_required=1
+case $(uname -m) in
+  aarch64)
+    binfmt_misc_required=0
+    ;;
+  arm*)
+    binfmt_misc_required=0
+    ;;
+esac
+
+# Check if qemu-aarch64-static and /proc/sys/fs/binfmt_misc are present
+if [[ "${binfmt_misc_required}" == "1" ]]; then
+  if ! qemu_arm=$(which qemu-aarch64-static) ; then
+    echo "qemu-aarch64-static not found (please install qemu-user-static)"
+    exit 1
+  fi
+  if [ ! -f /proc/sys/fs/binfmt_misc/register ]; then
+    echo "binfmt_misc required but not mounted, trying to mount it..."
+    if ! mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ; then
+        echo "mounting binfmt_misc failed"
+        exit 1
+    fi
+    echo "binfmt_misc mounted"
+  fi
+  if ! grep -q "^interpreter ${qemu_arm}" /proc/sys/fs/binfmt_misc/qemu-aarch64* ; then
+    # Register qemu-aarch64 for binfmt_misc
+    reg="echo ':qemu-aarch64-rpi:M::"\
+"\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00:"\
+"\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:"\
+"${qemu_arm}:F' > /proc/sys/fs/binfmt_misc/register"
+    echo "Registering qemu-aarch64 for binfmt_misc..."
+    sudo bash -c "${reg}" 2>/dev/null || true
+  fi
+fi
+
+trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${DOCKER_CMDLINE_NAME}' SIGINT SIGTERM
+time ${DOCKER} run \
+  $DOCKER_CMDLINE_PRE \
+  --name "${DOCKER_CMDLINE_NAME}" \
+  --privileged \
+  --cap-add=ALL \
+  -v /dev:/dev \
+  -v /lib/modules:/lib/modules \
+  ${PIGEN_DOCKER_OPTS} \
+  --volume "${CONFIG_FILE}":/config:ro \
+  -e "GIT_HASH=${GIT_HASH}" \
+  $DOCKER_CMDLINE_POST \
+  pi-gen \
+  bash -e -o pipefail -c "
+    dpkg-reconfigure qemu-user-static &&
+    # binfmt_misc is sometimes not mounted with debian bullseye image
+    (mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc || true) &&
+    cd /pi-gen; ./build.sh ${BUILD_OPTS} &&
+    rsync -av work/*/build.log deploy/
+  " &
+  wait "$!"
+
+# Ensure that deploy/ is always owned by calling user
+echo "copying results from deploy/"
+${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy - | tar -xf -
+
+echo "copying log from container ${CONTAINER_NAME} to deploy/"
+${DOCKER} logs --timestamps "${CONTAINER_NAME}" &>deploy/build-docker.log
+
+echo "copying kernel from container ${CONTAINER_NAME} to depoy/"
+${DOCKER} cp -q ${CONTAINER_NAME}:/pi-gen/stage2-rak/02-kernel/files/cm4.arm64.kernel.zip deploy/
+${DOCKER} cp -q ${CONTAINER_NAME}:/pi-gen/stage2-rak/02-kernel/files/rpi5.arm64.kernel.zip deploy/
+
+ls -lah deploy
 
 # cleanup
 if [ "${PRESERVE_CONTAINER}" != "1" ]; then
 	${DOCKER} rm -v "${CONTAINER_NAME}"
 fi
 
-echo "Done! Your image(s) should be in build/"
+echo "Done! Your image(s) should be in deploy/"

build.sh

@@ -21,7 +21,7 @@ EOF
 			PACKAGES="$(sed -f "${SCRIPT_DIR}/remove-comments.sed" < "${i}-packages-nr")"
 			if [ -n "$PACKAGES" ]; then
 				on_chroot << EOF
-apt-get -o APT::Acquire::Retries=3 install --no-install-recommends -y $PACKAGES
+apt-get -o Acquire::Retries=3 install --no-install-recommends -y $PACKAGES
 EOF
 				if [ "${USE_QCOW2}" = "1" ]; then
 					on_chroot << EOF
@@ -36,7 +36,7 @@ EOF
 			PACKAGES="$(sed -f "${SCRIPT_DIR}/remove-comments.sed" < "${i}-packages")"
 			if [ -n "$PACKAGES" ]; then
 				on_chroot << EOF
-apt-get -o APT::Acquire::Retries=3 install -y $PACKAGES
+apt-get -o Acquire::Retries=3 install -y $PACKAGES
 EOF
 				if [ "${USE_QCOW2}" = "1" ]; then
 					on_chroot << EOF
@@ -194,6 +194,7 @@ trap term EXIT INT TERM
 
 export PI_GEN=${PI_GEN:-pi-gen}
 export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen}
+export PI_GEN_RELEASE=${PI_GEN_RELEASE:-Raspberry Pi reference}
 
 if [ -z "${IMG_NAME}" ]; then
 	echo "IMG_NAME not set" 1>&2
@@ -226,9 +227,7 @@ export TARGET_HOSTNAME=${TARGET_HOSTNAME:-raspberrypi}
 export FIRST_USER_NAME=${FIRST_USER_NAME:-pi}
 export FIRST_USER_PASS
 export DISABLE_FIRST_BOOT_USER_RENAME=${DISABLE_FIRST_BOOT_USER_RENAME:-0}
-export RELEASE=${RELEASE:-bullseye}
-export WPA_ESSID
-export WPA_PASSWORD
+export RELEASE=${RELEASE:-bookworm} # Don't forget to update stage0/prerun.sh
 export WPA_COUNTRY
 export ENABLE_SSH="${ENABLE_SSH:-0}"
 export PUBKEY_ONLY_SSH="${PUBKEY_ONLY_SSH:-0}"
@@ -289,6 +288,10 @@ export KERNEL_BUILD=${KERNEL_BUILD:-0}
 export KERNEL_CACHED=${KERNEL_CACHED:-0}
 export KERNEL_TAG
 
+if [ "$SETFCAP" != "1" ]; then
+	export CAPSH_ARG="--drop=cap_setfcap"
+fi
+
 dependencies_check "${BASE_DIR}/depends"
 
 #check username is valid

config_rak

@@ -1,10 +1,12 @@
 ARCH=arm64
 NAME="rakpios"
-VERSION=0.7.1
+VERSION=0.8.0
+RELEASE=bookworm
 
+IMG_NAME="${NAME}-${VERSION}-${ARCH}"
+PI_GEN_RELEASE="${NAME}-${VERSION}-${RELEASE}"
 TARGET_HOSTNAME="${NAME}"
 IMG_DATE=$( date +%Y%m%d )
-IMG_NAME="${NAME}-${VERSION}-${ARCH}"
 ARCHIVE_FILENAME="${IMG_DATE}-${IMG_NAME}"
 FIRST_USER_NAME=rak
 FIRST_USER_PASS=changeme
@@ -15,4 +17,4 @@ STAGE_LIST="stage0 stage1 stage2 stage2-rak"
 PI_GEN_REPO=https://github.com/RAKWireless/rakpios
 KERNEL_BUILD=0
 KERNEL_CACHED=1
-KERNEL_TAG=rpi-5.15.y
+KERNEL_TAG=rpi-6.1.y

export-image/02-set-sources/01-run.sh

@@ -4,6 +4,6 @@ rm -f "${ROOTFS_DIR}/etc/apt/apt.conf.d/51cache"
 find "${ROOTFS_DIR}/var/lib/apt/lists/" -type f -delete
 on_chroot << EOF
 apt-get update
-apt-get -y dist-upgrade
+apt-get -y dist-upgrade --auto-remove --purge
 apt-get clean
 EOF

export-image/04-set-partuuid/00-run.sh

@@ -12,7 +12,6 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
 	sed -i "s/BOOTDEV/PARTUUID=${BOOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
 	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/etc/fstab"
 
-	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/cmdline.txt"
-
+	sed -i "s/ROOTDEV/PARTUUID=${ROOT_PARTUUID}/" "${ROOTFS_DIR}/boot/firmware/cmdline.txt"
 fi
 

export-image/05-finalise/01-run.sh

@@ -3,7 +3,10 @@
 IMG_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.img"
 INFO_FILE="${STAGE_WORK_DIR}/${IMG_FILENAME}${IMG_SUFFIX}.info"
 
+sed -i 's/^update_initramfs=.*/update_initramfs=all/' "${ROOTFS_DIR}/etc/initramfs-tools/update-initramfs.conf"
+
 on_chroot << EOF
+update-initramfs -u
 if [ -x /etc/init.d/fake-hwclock ]; then
 	/etc/init.d/fake-hwclock stop
 fi
@@ -53,7 +56,11 @@ rm -f "${ROOTFS_DIR}/root/.vnc/private.key"
 rm -f "${ROOTFS_DIR}/etc/vnc/updateid"
 
 update_issue "$(basename "${EXPORT_DIR}")"
-install -m 644 "${ROOTFS_DIR}/etc/rpi-issue" "${ROOTFS_DIR}/boot/issue.txt"
+install -m 644 "${ROOTFS_DIR}/etc/rpi-issue" "${ROOTFS_DIR}/boot/firmware/issue.txt"
+if ! [ -L "${ROOTFS_DIR}/boot/issue.txt" ]; then
+	ln -s firmware/issue.txt "${ROOTFS_DIR}/boot/issue.txt"
+fi
+
 
 cp "$ROOTFS_DIR/etc/rpi-issue" "$INFO_FILE"
 

export-image/prerun.sh

@@ -10,8 +10,8 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
 	rm -rf "${ROOTFS_DIR}"
 	mkdir -p "${ROOTFS_DIR}"
 
-	BOOT_SIZE="$((256 * 1024 * 1024))"
-	ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot --block-size=1 | cut -f 1)
+	BOOT_SIZE="$((512 * 1024 * 1024))"
+	ROOT_SIZE=$(du --apparent-size -s "${EXPORT_ROOTFS_DIR}" --exclude var/cache/apt/archives --exclude boot/firmware --block-size=1 | cut -f 1)
 
 	# All partition sizes and starts will be aligned to this size
 	ALIGN="$((4 * 1024 * 1024))"
@@ -33,55 +33,35 @@ if [ "${NO_PRERUN_QCOW2}" = "0" ]; then
 	parted --script "${IMG_FILE}" unit B mkpart primary fat32 "${BOOT_PART_START}" "$((BOOT_PART_START + BOOT_PART_SIZE - 1))"
 	parted --script "${IMG_FILE}" unit B mkpart primary ext4 "${ROOT_PART_START}" "$((ROOT_PART_START + ROOT_PART_SIZE - 1))"
 
-	PARTED_OUT=$(parted -sm "${IMG_FILE}" unit b print)
-	BOOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 2 | tr -d B)
-	BOOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^1:' | cut -d':' -f 4 | tr -d B)
-
-	ROOT_OFFSET=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 2 | tr -d B)
-	ROOT_LENGTH=$(echo "$PARTED_OUT" | grep -e '^2:' | cut -d':' -f 4 | tr -d B)
-
-	echo "Mounting BOOT_DEV..."
-	cnt=0
-	until BOOT_DEV=$(losetup --show -f -o "${BOOT_OFFSET}" --sizelimit "${BOOT_LENGTH}" "${IMG_FILE}"); do
-		if [ $cnt -lt 5 ]; then
-			cnt=$((cnt + 1))
-			echo "Error in losetup for BOOT_DEV.  Retrying..."
-			sleep 5
-		else
-			echo "ERROR: losetup for BOOT_DEV failed; exiting"
-			exit 1
-		fi
-	done
-
-	echo "Mounting ROOT_DEV..."
+	echo "Creating loop device..."
 	cnt=0
-	until ROOT_DEV=$(losetup --show -f -o "${ROOT_OFFSET}" --sizelimit "${ROOT_LENGTH}" "${IMG_FILE}"); do
+	until ensure_next_loopdev && LOOP_DEV="$(losetup --show --find --partscan "$IMG_FILE")"; do
 		if [ $cnt -lt 5 ]; then
 			cnt=$((cnt + 1))
-			echo "Error in losetup for ROOT_DEV.  Retrying..."
+			echo "Error in losetup.  Retrying..."
 			sleep 5
 		else
-			echo "ERROR: losetup for ROOT_DEV failed; exiting"
+			echo "ERROR: losetup failed; exiting"
 			exit 1
 		fi
 	done
 
-	echo "/boot: offset $BOOT_OFFSET, length $BOOT_LENGTH"
-	echo "/:     offset $ROOT_OFFSET, length $ROOT_LENGTH"
+	BOOT_DEV="${LOOP_DEV}p1"
+	ROOT_DEV="${LOOP_DEV}p2"
 
 	ROOT_FEATURES="^huge_file"
 	for FEATURE in 64bit; do
 	if grep -q "$FEATURE" /etc/mke2fs.conf; then
 		ROOT_FEATURES="^$FEATURE,$ROOT_FEATURES"
 	fi
 	done
-	mkdosfs -n boot -F 32 -s 4 -v "$BOOT_DEV" > /dev/null
+	mkdosfs -n bootfs -F 32 -s 4 -v "$BOOT_DEV" > /dev/null
 	mkfs.ext4 -L rootfs -O "$ROOT_FEATURES" "$ROOT_DEV" > /dev/null
 
 	mount -v "$ROOT_DEV" "${ROOTFS_DIR}" -t ext4
-	mkdir -p "${ROOTFS_DIR}/boot"
-	mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot" -t vfat
+	mkdir -p "${ROOTFS_DIR}/boot/firmware"
+	mount -v "$BOOT_DEV" "${ROOTFS_DIR}/boot/firmware" -t vfat
 
-	rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/"
-	rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/" "${ROOTFS_DIR}/boot/"
+	rsync -aHAXx --exclude /var/cache/apt/archives --exclude /boot/firmware "${EXPORT_ROOTFS_DIR}/" "${ROOTFS_DIR}/"
+	rsync -rtx "${EXPORT_ROOTFS_DIR}/boot/firmware/" "${ROOTFS_DIR}/boot/firmware/"
 fi