Skip to content

Commit

Permalink
neonvm-runner: modify qemu arguments to run on arm64
Browse files Browse the repository at this point in the history
  • Loading branch information
mikhail-sakhnov committed Oct 21, 2024
1 parent 956d57f commit 4915d7c
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 17 deletions.
2 changes: 1 addition & 1 deletion neonvm-runner/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ COPY . .
# Build
RUN CGO_ENABLED=0 go build -o /runner neonvm-runner/cmd/*.go


FROM alpine:3.19

RUN apk add --no-cache \
Expand All @@ -21,6 +20,7 @@ RUN apk add --no-cache \
busybox-extras \
e2fsprogs \
qemu-system-x86_64 \
qemu-system-aarch64 \
qemu-img \
cgroup-tools \
openssh
Expand Down
57 changes: 43 additions & 14 deletions neonvm-runner/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"os/signal"
"path/filepath"
"regexp"
"runtime"
"strings"
"sync"
"sync/atomic"
Expand Down Expand Up @@ -48,8 +49,12 @@ import (
)

const (
QEMU_BIN = "qemu-system-x86_64"
QEMU_IMG_BIN = "qemu-img"
qemuBinArm64 = "qemu-system-aarch64"
qemuBinX8664 = "qemu-system-x86_64"
qemuImgBin = "qemu-img"

architectureArm64 = "arm64"
architectureAmd64 = "amd64"
defaultKernelPath = "/vm/kernel/vmlinuz"

rootDiskPath = "/vm/images/rootdisk.qcow2"
Expand Down Expand Up @@ -397,14 +402,14 @@ func calcDirUsage(dirPath string) (int64, error) {
func createSwap(diskPath string, swapSize *resource.Quantity) error {
tmpRawFile := "swap.raw"

if err := execFg(QEMU_IMG_BIN, "create", "-q", "-f", "raw", tmpRawFile, fmt.Sprintf("%d", swapSize.Value())); err != nil {
if err := execFg(qemuImgBin, "create", "-q", "-f", "raw", tmpRawFile, fmt.Sprintf("%d", swapSize.Value())); err != nil {
return err
}
if err := execFg("mkswap", "-L", swapName, tmpRawFile); err != nil {
return err
}

if err := execFg(QEMU_IMG_BIN, "convert", "-q", "-f", "raw", "-O", "qcow2", "-o", "cluster_size=2M,lazy_refcounts=on", tmpRawFile, diskPath); err != nil {
if err := execFg(qemuImgBin, "convert", "-q", "-f", "raw", "-O", "qcow2", "-o", "cluster_size=2M,lazy_refcounts=on", tmpRawFile, diskPath); err != nil {
return err
}

Expand Down Expand Up @@ -460,7 +465,7 @@ func createQCOW2(diskName string, diskPath string, diskSize *resource.Quantity,
return err
}

if err := execFg(QEMU_IMG_BIN, "convert", "-q", "-f", "raw", "-O", "qcow2", "-o", "cluster_size=2M,lazy_refcounts=on", "ext4.raw", diskPath); err != nil {
if err := execFg(qemuImgBin, "convert", "-q", "-f", "raw", "-O", "qcow2", "-o", "cluster_size=2M,lazy_refcounts=on", "ext4.raw", diskPath); err != nil {
return err
}

Expand Down Expand Up @@ -615,6 +620,7 @@ type Config struct {
memoryProvider vmv1.MemoryProvider
autoMovableRatio string
cpuScalingMode string
architecture string
}

func newConfig(logger *zap.Logger) *Config {
Expand All @@ -625,9 +631,10 @@ func newConfig(logger *zap.Logger) *Config {
appendKernelCmdline: "",
skipCgroupManagement: false,
diskCacheSettings: "cache=none",
memoryProvider: "", // Require that this is explicitly set. We'll check later.
autoMovableRatio: "", // Require that this is explicitly set IFF memoryProvider is VirtioMem. We'll check later.
cpuScalingMode: "", // Require that this is explicitly set. We'll check later.
memoryProvider: "", // Require that this is explicitly set. We'll check later.
autoMovableRatio: "", // Require that this is explicitly set IFF memoryProvider is VirtioMem. We'll check later.
cpuScalingMode: "", // Require that this is explicitly set. We'll check later.
architecture: runtime.GOARCH, // arm64, amd64
}
flag.StringVar(&cfg.vmSpecDump, "vmspec", cfg.vmSpecDump,
"Base64 encoded VirtualMachine json specification")
Expand Down Expand Up @@ -777,7 +784,7 @@ func resizeRootDisk(logger *zap.Logger, vmSpec *vmv1.VirtualMachineSpec) error {
VirtualSize int64 `json:"virtual-size"`
}
// get current disk size by qemu-img info command
qemuImgOut, err := exec.Command(QEMU_IMG_BIN, "info", "--output=json", rootDiskPath).Output()
qemuImgOut, err := exec.Command(qemuImgBin, "info", "--output=json", rootDiskPath).Output()
if err != nil {
return fmt.Errorf("could not get root image size: %w", err)
}
Expand All @@ -791,7 +798,7 @@ func resizeRootDisk(logger *zap.Logger, vmSpec *vmv1.VirtualMachineSpec) error {
if !vmSpec.Guest.RootDisk.Size.IsZero() {
if vmSpec.Guest.RootDisk.Size.Cmp(*imageSizeQuantity) == 1 {
logger.Info(fmt.Sprintf("resizing rootDisk from %s to %s", imageSizeQuantity.String(), vmSpec.Guest.RootDisk.Size.String()))
if err := execFg(QEMU_IMG_BIN, "resize", rootDiskPath, fmt.Sprintf("%d", vmSpec.Guest.RootDisk.Size.Value())); err != nil {
if err := execFg(qemuImgBin, "resize", rootDiskPath, fmt.Sprintf("%d", vmSpec.Guest.RootDisk.Size.Value())); err != nil {
return fmt.Errorf("failed to resize rootDisk: %w", err)
}
} else {
Expand All @@ -813,7 +820,7 @@ func buildQEMUCmd(
// prepare qemu command line
qemuCmd := []string{
"-runas", "qemu",
"-machine", "q35",
"-machine", getMachineType(cfg.architecture),
"-nographic",
"-no-reboot",
"-nodefaults",
Expand Down Expand Up @@ -1050,7 +1057,6 @@ func runQEMU(

wg.Add(1)
go terminateQemuOnSigterm(ctx, logger, &wg)

var callbacks cpuServerCallbacks

lastValue := &atomic.Uint32{}
Expand Down Expand Up @@ -1078,13 +1084,14 @@ func runQEMU(
wg.Add(1)
go forwardLogs(ctx, logger, &wg)

qemuBin := getQemuBinaryName(cfg.architecture)
var bin string
var cmd []string
if !cfg.skipCgroupManagement {
bin = "cgexec"
cmd = append([]string{"-g", fmt.Sprintf("cpu:%s", cgroupPath), QEMU_BIN}, qemuCmd...)
cmd = append([]string{"-g", fmt.Sprintf("cpu:%s", cgroupPath), qemuBin}, qemuCmd...)
} else {
bin = QEMU_BIN
bin = qemuBin
cmd = qemuCmd
}

Expand All @@ -1104,6 +1111,28 @@ func runQEMU(
return err
}

func getQemuBinaryName(architecture string) string {
switch architecture {
case architectureArm64:
return qemuBinArm64
case architectureAmd64:
return qemuBinX8664
default:
panic(fmt.Errorf("unknown architecture %s", architecture))
}
}

func getMachineType(architecture string) string {
switch architecture {
case architectureArm64:
return "virt"
case architectureAmd64:
return "q35"
default:
panic(fmt.Errorf("unknown architecture %s", architecture))
}
}

func handleCPUChange(
logger *zap.Logger,
w http.ResponseWriter,
Expand Down
4 changes: 2 additions & 2 deletions pkg/neonvm/controllers/vm_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"net/http"
"os"
"reflect"
sysruntime "runtime"
"strconv"
"time"

Expand Down Expand Up @@ -1215,7 +1216,6 @@ func affinityForVirtualMachine(vm *vmv1.VirtualMachine) *corev1.Affinity {
if a.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
a.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{}
}

// if NodeSelectorTerms list is empty - add default values (arch==amd64 or os==linux)
if len(a.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms) == 0 {
a.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(
Expand All @@ -1225,7 +1225,7 @@ func affinityForVirtualMachine(vm *vmv1.VirtualMachine) *corev1.Affinity {
{
Key: "kubernetes.io/arch",
Operator: "In",
Values: []string{"amd64"},
Values: []string{sysruntime.GOARCH},
},
{
Key: "kubernetes.io/os",
Expand Down

0 comments on commit 4915d7c

Please sign in to comment.