diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index d20e92ad..8dbb05ed 100755 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -1,4 +1,4 @@ #!/bin/bash set -euo pipefail -find . -user root -print0 | xargs -0 sudo rm -rf '{}' +sudo chown -hR "${USER}:${USER}" . diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a53a28a9..7d713e02 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -78,6 +78,9 @@ steps: - ln -s /var/lib/fc-ci/rootfs.ext4 ${FC_TEST_DATA_PATH}/root-drive.img # Download Firecracker and its jailer. - make deps + # Build a rootfs with SSH enabled. + - sudo -E FC_TEST_DATA_PATH=${FC_TEST_DATA_PATH} make ${FC_TEST_DATA_PATH}/root-drive-ssh-key + - stat ${FC_TEST_DATA_PATH}/root-drive-ssh-key ${FC_TEST_DATA_PATH}/root-drive-with-ssh.img agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}" distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}" @@ -110,7 +113,7 @@ steps: - label: ':hammer: root tests' commands: - - "sudo PATH=$PATH FC_TEST_TAP=fc-root-tap${BUILDKITE_BUILD_NUMBER} FC_TEST_DATA_PATH=${FC_TEST_DATA_PATH} make test EXTRAGOARGS='-v -count=1 -race' DISABLE_ROOT_TESTS=" + - "sudo -E PATH=$PATH FC_TEST_TAP=fc-root-tap${BUILDKITE_BUILD_NUMBER} FC_TEST_DATA_PATH=${FC_TEST_DATA_PATH} make test EXTRAGOARGS='-v -count=1 -race' DISABLE_ROOT_TESTS=" agents: queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}" distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}" diff --git a/.gitignore b/.gitignore index 5ed11328..e4241aad 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ root-drive.img TestPID.img build/ testdata/fc.stamp +testdata/bin/ testdata/logs/ testdata/ltag testdata/release-* diff --git a/Makefile b/Makefile index 260fc81e..0973a4b7 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ FIRECRACKER_DIR=build/firecracker FIRECRACKER_TARGET?=x86_64-unknown-linux-musl FC_TEST_DATA_PATH?=testdata +FC_TEST_BIN_PATH:=$(FC_TEST_DATA_PATH)/bin FIRECRACKER_BIN=$(FC_TEST_DATA_PATH)/firecracker-main JAILER_BIN=$(FC_TEST_DATA_PATH)/jailer-main @@ -37,7 +38,17 @@ $(FC_TEST_DATA_PATH)/vmlinux \ $(FC_TEST_DATA_PATH)/root-drive.img \ $(FC_TEST_DATA_PATH)/jailer \ $(FC_TEST_DATA_PATH)/firecracker \ -$(FC_TEST_DATA_PATH)/ltag +$(FC_TEST_DATA_PATH)/ltag \ +$(FC_TEST_BIN_PATH)/ptp \ +$(FC_TEST_BIN_PATH)/host-local \ +$(FC_TEST_BIN_PATH)/static \ +$(FC_TEST_BIN_PATH)/tc-redirect-tap + +# Enable pulling of artifacts from S3 instead of building +# TODO: https://github.com/firecracker-microvm/firecracker-go-sdk/issues/418 +ifeq ($(GID), 0) +testdata_objects += $(FC_TEST_DATA_PATH)/root-drive-with-ssh.img $(FC_TEST_DATA_PATH)/root-drive-ssh-key +endif testdata_dir = testdata/firecracker.tgz testdata/firecracker_spec-$(firecracker_version).yaml testdata/LICENSE testdata/NOTICE testdata/THIRD-PARTY @@ -82,6 +93,34 @@ $(FC_TEST_DATA_PATH)/fc.stamp: $(FC_TEST_DATA_PATH)/root-drive.img: $(curl) -o $@ https://s3.amazonaws.com/spec.ccfc.min/img/hello/fsfiles/hello-rootfs.ext4 +$(FC_TEST_DATA_PATH)/root-drive-ssh-key $(FC_TEST_DATA_PATH)/root-drive-with-ssh.img: +# Need root to move ssh key to testdata location +ifeq ($(GID), 0) + $(MAKE) $(FIRECRACKER_DIR) + $(FIRECRACKER_DIR)/tools/devtool build_rootfs + cp $(FIRECRACKER_DIR)/build/rootfs/bionic.rootfs.ext4 $(FC_TEST_DATA_PATH)/root-drive-with-ssh.img + cp $(FIRECRACKER_DIR)/build/rootfs/ssh/id_rsa $(FC_TEST_DATA_PATH)/root-drive-ssh-key + rm -rf $(FIRECRACKER_DIR) +else + $(error unable to place ssh key without root permissions) +endif + +$(FC_TEST_BIN_PATH)/ptp: + GO111MODULE=off GOBIN=$(abspath $(FC_TEST_BIN_PATH)) \ + go get github.com/containernetworking/plugins/plugins/main/ptp + +$(FC_TEST_BIN_PATH)/host-local: + GO111MODULE=off GOBIN=$(abspath $(FC_TEST_BIN_PATH)) \ + go get github.com/containernetworking/plugins/plugins/ipam/host-local + +$(FC_TEST_BIN_PATH)/static: + GO111MODULE=off GOBIN=$(abspath $(FC_TEST_BIN_PATH)) \ + go get github.com/containernetworking/plugins/plugins/ipam/static + +$(FC_TEST_BIN_PATH)/tc-redirect-tap: + GO111MODULE=off GOBIN=$(abspath $(FC_TEST_BIN_PATH)) \ + go get github.com/awslabs/tc-redirect-tap/cmd/tc-redirect-tap + $(FC_TEST_DATA_PATH)/ltag: GO111MODULE=off GOBIN=$(abspath $(FC_TEST_DATA_PATH)) \ go get github.com/kunalkushwaha/ltag diff --git a/firecracker.go b/firecracker.go index 3102400a..0df8b55b 100644 --- a/firecracker.go +++ b/firecracker.go @@ -281,6 +281,23 @@ func (f *Client) CreateSnapshot(ctx context.Context, snapshotParams *models.Snap return f.client.Operations.CreateSnapshot(params) } +// LoadSnapshotOpt is a functional option to be used for the +// LoadSnapshot API in setting any additional optional fields. +type LoadSnapshotOpt func(*ops.LoadSnapshotParams) + +// LoadSnapshot is a wrapper for the swagger generated client to make +// calling of the API easier. +func (f *Client) LoadSnapshot(ctx context.Context, snapshotParams *models.SnapshotLoadParams, opts ...LoadSnapshotOpt) (*ops.LoadSnapshotNoContent, error) { + params := ops.NewLoadSnapshotParamsWithContext(ctx) + params.SetBody(snapshotParams) + + for _, opt := range opts { + opt(params) + } + + return f.client.Operations.LoadSnapshot(params) +} + // CreateSyncActionOpt is a functional option to be used for the // CreateSyncAction API in setting any additional optional fields. type CreateSyncActionOpt func(*ops.CreateSyncActionParams) diff --git a/go.mod b/go.mod index 3bd93205..c26b7998 100644 --- a/go.mod +++ b/go.mod @@ -18,5 +18,6 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.8.0 github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 + golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a ) diff --git a/go.sum b/go.sum index f0d4c8db..39b69e49 100644 --- a/go.sum +++ b/go.sum @@ -724,6 +724,8 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -792,6 +794,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -886,6 +889,7 @@ golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a h1:ppl5mZgokTT8uPkmYOyEUmPTr golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/handlers.go b/handlers.go index 24013836..66906f99 100644 --- a/handlers.go +++ b/handlers.go @@ -34,7 +34,8 @@ const ( LinkFilesToRootFSHandlerName = "fcinit.LinkFilesToRootFS" SetupNetworkHandlerName = "fcinit.SetupNetwork" SetupKernelArgsHandlerName = "fcinit.SetupKernelArgs" - CreateBalloonHandlerName = "fcint.CreateBalloon" + CreateBalloonHandlerName = "fcinit.CreateBalloon" + LoadSnapshotHandlerName = "fcinit.LoadSnapshot" ValidateCfgHandlerName = "validate.Cfg" ValidateJailerCfgHandlerName = "validate.JailerCfg" @@ -280,6 +281,15 @@ func NewCreateBalloonHandler(amountMib int64, deflateOnOom bool, StatsPollingInt } } +// LoadSnapshotHandler is a named handler that loads a snapshot +// from the specified filepath +var LoadSnapshotHandler = Handler{ + Name: LoadSnapshotHandlerName, + Fn: func(ctx context.Context, m *Machine) error { + return m.loadSnapshot(ctx, &m.Cfg.Snapshot) + }, +} + var defaultFcInitHandlerList = HandlerList{}.Append( SetupNetworkHandler, SetupKernelArgsHandler, @@ -294,6 +304,15 @@ var defaultFcInitHandlerList = HandlerList{}.Append( ConfigMmdsHandler, ) +var loadSnapshotHandlerList = HandlerList{}.Append( + SetupNetworkHandler, + StartVMMHandler, + CreateLogFilesHandler, + BootstrapLoggingHandler, + LoadSnapshotHandler, + AddVsocksHandler, +) + var defaultValidationHandlerList = HandlerList{}.Append( NetworkConfigValidationHandler, ) diff --git a/machine.go b/machine.go index 31f85dba..8f1dfd86 100644 --- a/machine.go +++ b/machine.go @@ -151,6 +151,13 @@ type Config struct { // It is possible to use a valid IPv4 link-local address (169.254.0.0/16). // If not provided, the default address (169.254.169.254) will be used. MmdsAddress net.IP + + // Configuration for snapshot loading + Snapshot SnapshotConfig +} + +func (cfg *Config) hasSnapshot() bool { + return cfg.Snapshot.MemFilePath != "" || cfg.Snapshot.SnapshotPath != "" } // Validate will ensure that the required fields are set and that @@ -381,7 +388,7 @@ func NewMachine(ctx context.Context, cfg Config, opts ...Opt) (*Machine, error) // handlers succeed, then this will start the VMM instance. // Start may only be called once per Machine. Subsequent calls will return // ErrAlreadyStarted. -func (m *Machine) Start(ctx context.Context) error { +func (m *Machine) Start(ctx context.Context, opts ...StartOpt) error { m.logger.Debug("Called Machine.Start()") alreadyStarted := true m.startOnce.Do(func() { @@ -402,6 +409,10 @@ func (m *Machine) Start(ctx context.Context) error { } }() + for _, opt := range opts { + opt(m) + } + err = m.Handlers.Run(ctx, m) if err != nil { return err @@ -854,6 +865,10 @@ func (m *Machine) addVsock(ctx context.Context, dev VsockDevice) error { } func (m *Machine) startInstance(ctx context.Context) error { + if m.Cfg.hasSnapshot() { + return nil + } + action := models.InstanceActionInfoActionTypeInstanceStart info := models.InstanceActionInfo{ ActionType: &action, @@ -1105,6 +1120,23 @@ func (m *Machine) CreateSnapshot(ctx context.Context, memFilePath, snapshotPath return nil } +// loadSnapshot loads a snapshot of the VM +func (m *Machine) loadSnapshot(ctx context.Context, snapshot *SnapshotConfig) error { + snapshotParams := &models.SnapshotLoadParams{ + MemFilePath: &snapshot.MemFilePath, + SnapshotPath: &snapshot.SnapshotPath, + EnableDiffSnapshots: snapshot.EnableDiffSnapshots, + ResumeVM: snapshot.ResumeVM, + } + + if _, err := m.client.LoadSnapshot(ctx, snapshotParams); err != nil { + return fmt.Errorf("failed to load a snapshot for VM: %v", err) + } + + m.logger.Debug("snapshot loaded successfully") + return nil +} + // CreateBalloon creates a balloon device if one does not exist func (m *Machine) CreateBalloon(ctx context.Context, amountMib int64, deflateOnOom bool, statsPollingIntervals int64, opts ...PutBalloonOpt) error { balloon := models.Balloon{ diff --git a/machine_test.go b/machine_test.go index 5a72571c..24583428 100644 --- a/machine_test.go +++ b/machine_test.go @@ -38,6 +38,7 @@ import ( "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/crypto/ssh" models "github.com/firecracker-microvm/firecracker-go-sdk/client/models" ops "github.com/firecracker-microvm/firecracker-go-sdk/client/operations" @@ -66,7 +67,9 @@ var ( testDataLogPath = filepath.Join(testDataPath, "logs") testDataBin = filepath.Join(testDataPath, "bin") - testRootfs = filepath.Join(testDataPath, "root-drive.img") + testRootfs = filepath.Join(testDataPath, "root-drive.img") + testRootfsWithSSH = filepath.Join(testDataPath, "root-drive-with-ssh.img") + testSSHKey = filepath.Join(testDataPath, "root-drive-ssh-key") testBalloonMemory = int64(10) testBalloonNewMemory = int64(6) @@ -1467,8 +1470,48 @@ func TestWaitWithNoSocket(t *testing.T) { } } -func createValidConfig(t *testing.T, socketPath string) Config { - return Config{ +type machineConfigOpt func(c *Config) + +func withRootDrive(rootfs string) machineConfigOpt { + return func(c *Config) { + var drives []models.Drive + + inserted := false + for _, drive := range c.Drives { + if *drive.IsRootDevice { + drives = append(drives, models.Drive{ + DriveID: String("root"), + IsRootDevice: Bool(true), + IsReadOnly: Bool(false), + PathOnHost: String(rootfs), + }) + inserted = true + } else { + drives = append(drives, drive) + } + } + + if !inserted { + drives = append(drives, models.Drive{ + DriveID: String("root"), + IsRootDevice: Bool(true), + IsReadOnly: Bool(false), + PathOnHost: String(rootfs), + }) + } + + c.Drives = drives + } +} + +func withNetworkInterface(networkInterface NetworkInterface) machineConfigOpt { + return func(c *Config) { + c.NetworkInterfaces = append(c.NetworkInterfaces, networkInterface) + } +} + +func createValidConfig(t *testing.T, socketPath string, opts ...machineConfigOpt) Config { + cfg := Config{ SocketPath: socketPath, KernelImagePath: getVmlinuxPath(t), MachineCfg: models.MachineConfiguration{ @@ -1486,6 +1529,12 @@ func createValidConfig(t *testing.T, socketPath string) Config { }, }, } + + for _, opt := range opts { + opt(&cfg) + } + + return cfg } func TestSignalForwarding(t *testing.T) { @@ -1772,6 +1821,330 @@ func TestCreateSnapshot(t *testing.T) { } } +func connectToVM(m *Machine, sshKeyPath string) (*ssh.Client, error) { + key, err := ioutil.ReadFile(sshKeyPath) + if err != nil { + return nil, err + } + + signer, err := ssh.ParsePrivateKey(key) + if err != nil { + return nil, err + } + + config := &ssh.ClientConfig{ + User: "root", + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: 5 * time.Second, + } + + if len(m.Cfg.NetworkInterfaces) == 0 { + return nil, errors.New("No network interfaces") + } + + ip := m.Cfg.NetworkInterfaces.staticIPInterface().StaticConfiguration.IPConfiguration.IPAddr.IP + + return ssh.Dial("tcp", fmt.Sprintf("%s:22", ip), config) +} + +func writeCNIConfWithHostLocalSubnet(path, networkName, subnet string) error { + return ioutil.WriteFile(path, []byte(fmt.Sprintf(`{ + "cniVersion": "0.3.1", + "name": "%s", + "plugins": [ + { + "type": "ptp", + "ipam": { + "type": "host-local", + "subnet": "%s" + } + }, + { + "type": "tc-redirect-tap" + } + ] + }`, networkName, subnet)), 0644) +} + +func TestLoadSnapshot(t *testing.T) { + fctesting.RequiresKVM(t) + fctesting.RequiresRoot(t) + + dir, err := ioutil.TempDir("", t.Name()) + require.NoError(t, err) + defer os.RemoveAll(dir) + + cniConfDir := filepath.Join(dir, "cni.conf") + err = os.MkdirAll(cniConfDir, 0777) + require.NoError(t, err) + + cniBinPath := []string{testDataBin} + + rootfsBytes, err := ioutil.ReadFile(testRootfsWithSSH) + require.NoError(t, err) + rootfsPath := filepath.Join(dir, "rootfs.img") + err = ioutil.WriteFile(rootfsPath, rootfsBytes, 0666) + require.NoError(t, err) + + sshKeyBytes, err := ioutil.ReadFile(testSSHKey) + require.NoError(t, err) + sshKeyPath := filepath.Join(dir, "id_rsa") + err = ioutil.WriteFile(sshKeyPath, sshKeyBytes, 0600) + require.NoError(t, err) + + // Using default cache directory to ensure collision avoidance on IP allocations + const cniCacheDir = "/var/lib/cni" + const networkName = "fcnet" + const ifName = "veth0" + + networkMask := "/24" + subnet := "10.168.0.0" + networkMask + var ipToRestore string + + var maxRetries int = 20 + var backoffTimeMs time.Duration = 500 + + cases := []struct { + name string + createSnapshot func(ctx context.Context, machineLogger *logrus.Logger, socketPath, memPath, snapPath string) + loadSnapshot func(ctx context.Context, machineLogger *logrus.Logger, socketPath, memPath, snapPath string) + }{ + { + name: "TestLoadSnapshot", + createSnapshot: func(ctx context.Context, machineLogger *logrus.Logger, socketPath, memPath, snapPath string) { + // Create a snapshot + cfg := createValidConfig(t, socketPath+".create") + m, err := NewMachine(ctx, cfg, func(m *Machine) { + // Rewriting m.cmd partially wouldn't work since Cmd has + // some unexported members + args := m.cmd.Args[1:] + m.cmd = exec.Command(getFirecrackerBinaryPath(), args...) + }, WithLogger(logrus.NewEntry(machineLogger))) + require.NoError(t, err) + + err = m.Start(ctx) + require.NoError(t, err) + + err = m.PauseVM(ctx) + require.NoError(t, err) + + err = m.CreateSnapshot(ctx, memPath, snapPath) + require.NoError(t, err) + + err = m.StopVMM() + require.NoError(t, err) + }, + + loadSnapshot: func(ctx context.Context, machineLogger *logrus.Logger, socketPath, memPath, snapPath string) { + cfg := createValidConfig(t, socketPath+".load") + m, err := NewMachine(ctx, cfg, func(m *Machine) { + // Rewriting m.cmd partially wouldn't work since Cmd has + // some unexported members + args := m.cmd.Args[1:] + m.cmd = exec.Command(getFirecrackerBinaryPath(), args...) + }, WithLogger(logrus.NewEntry(machineLogger))) + require.NoError(t, err) + + err = m.Start(ctx, WithSnapshot(memPath, snapPath)) + require.NoError(t, err) + + err = m.ResumeVM(ctx) + require.NoError(t, err) + + err = m.StopVMM() + require.NoError(t, err) + }, + }, + { + name: "TestLoadSnapshot without create", + createSnapshot: func(ctx context.Context, machineLogger *logrus.Logger, socketPath, memPath, snapPath string) { + + }, + + loadSnapshot: func(ctx context.Context, machineLogger *logrus.Logger, socketPath, memPath, snapPath string) { + cfg := createValidConfig(t, socketPath+".load") + m, err := NewMachine(ctx, cfg, func(m *Machine) { + // Rewriting m.cmd partially wouldn't work since Cmd has + // some unexported members + args := m.cmd.Args[1:] + m.cmd = exec.Command(getFirecrackerBinaryPath(), args...) + }, WithLogger(logrus.NewEntry(machineLogger))) + require.NoError(t, err) + + err = m.Start(ctx, WithSnapshot(memPath, snapPath)) + require.Error(t, err) + }, + }, + { + name: "TestLoadSnapshot and check contents (via ssh)", + createSnapshot: func(ctx context.Context, machineLogger *logrus.Logger, socketPath, memPath, snapPath string) { + cniConfPath := fmt.Sprintf("%s/%s.conflist", cniConfDir, networkName) + err := writeCNIConfWithHostLocalSubnet(cniConfPath, networkName, subnet) + require.NoError(t, err) + defer os.Remove(cniConfPath) + + networkInterface := NetworkInterface{ + CNIConfiguration: &CNIConfiguration{ + NetworkName: networkName, + IfName: ifName, + ConfDir: cniConfDir, + BinPath: cniBinPath, + VMIfName: "eth0", + }, + } + cfg := createValidConfig(t, fmt.Sprintf("%s.create", socketPath), + withRootDrive(rootfsPath), + withNetworkInterface(networkInterface), + ) + + cmd := VMCommandBuilder{}.WithSocketPath(fmt.Sprintf("%s.create", socketPath)).WithBin(getFirecrackerBinaryPath()).Build(ctx) + + m, err := NewMachine(ctx, cfg, WithProcessRunner(cmd)) + require.NoError(t, err) + + err = m.Start(ctx) + require.NoError(t, err) + defer m.StopVMM() + defer func() { + if err := m.Shutdown(ctx); err != nil { + t.Logf("error on vm shutdown: %v", err) + } + }() + + var client *ssh.Client + for i := 0; i < maxRetries; i++ { + client, err = connectToVM(m, sshKeyPath) + if err != nil { + time.Sleep(backoffTimeMs * time.Millisecond) + } else { + break + } + } + require.NoError(t, err) + defer client.Close() + + ipToRestore = m.Cfg.NetworkInterfaces.staticIPInterface().StaticConfiguration.IPConfiguration.IPAddr.IP.String() + + session, err := client.NewSession() + require.NoError(t, err) + defer session.Close() + + err = session.Start(`sleep 422`) // Should be asynchronous + require.NoError(t, err) + + err = m.PauseVM(ctx) + require.NoError(t, err) + + err = m.CreateSnapshot(ctx, memPath, snapPath) + require.NoError(t, err) + }, + + loadSnapshot: func(ctx context.Context, machineLogger *logrus.Logger, socketPath, memPath, snapPath string) { + var ipFreed bool = false + var err error + + for i := 0; i < maxRetries; i++ { + // Wait till the file no longer exists (i.e. os.Stat returns an error) + if _, err = os.Stat(fmt.Sprintf("%s/networks/%s/%s", cniCacheDir, networkName, ipToRestore)); err == nil { + time.Sleep(backoffTimeMs * time.Millisecond) + } else { + ipFreed = true + break + } + } + + if errors.Is(err, os.ErrNotExist) { + err = nil + } else if !ipFreed { + err = fmt.Errorf("IP %v was not freed", ipToRestore) + } + require.NoError(t, err) + + cniConfPath := fmt.Sprintf("%s/%s.conflist", cniConfDir, networkName) + err = writeCNIConfWithHostLocalSubnet(cniConfPath, networkName, subnet) + require.NoError(t, err) + defer os.Remove(cniConfPath) + + networkInterface := NetworkInterface{ + CNIConfiguration: &CNIConfiguration{ + NetworkName: networkName, + IfName: ifName, + ConfDir: cniConfDir, + BinPath: cniBinPath, + Args: [][2]string{{"IP", ipToRestore + networkMask}}, + VMIfName: "eth0", + }, + } + cfg := createValidConfig(t, fmt.Sprintf("%s.load", socketPath), + withRootDrive(rootfsPath), + withNetworkInterface(networkInterface), + ) + + cmd := VMCommandBuilder{}.WithSocketPath(fmt.Sprintf("%s.load", socketPath)).WithBin(getFirecrackerBinaryPath()).Build(ctx) + + m, err := NewMachine(ctx, cfg, WithProcessRunner(cmd)) + require.NoError(t, err) + + err = m.Start(ctx, WithSnapshot(memPath, snapPath)) + require.NoError(t, err) + defer m.StopVMM() + + err = m.ResumeVM(ctx) + require.NoError(t, err) + + var client *ssh.Client + + for i := 0; i < maxRetries; i++ { + client, err = connectToVM(m, sshKeyPath) + if err != nil { + time.Sleep(backoffTimeMs * time.Millisecond) + } else { + break + } + } + require.NoError(t, err) + defer client.Close() + + session, err := client.NewSession() + require.NoError(t, err) + defer session.Close() + + var b bytes.Buffer + session.Stdout = &b + err = session.Run(`ps -aux | grep "sleep 422" | wc -l`) + require.NoError(t, err) + require.Equal(t, "3\n", b.String()) + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ctx := context.Background() + + // Set snap and mem paths + socketPath := filepath.Join(dir, fsSafeTestName.Replace(t.Name())) + snapPath := socketPath + "SnapFile" + memPath := socketPath + "MemFile" + defer os.Remove(socketPath) + defer os.Remove(snapPath) + defer os.Remove(memPath) + + // Tee logs for validation: + var logBuffer bytes.Buffer + machineLogger := logrus.New() + machineLogger.Out = io.MultiWriter(os.Stderr, &logBuffer) + + c.createSnapshot(ctx, machineLogger, socketPath, snapPath, memPath) + c.loadSnapshot(ctx, machineLogger, socketPath, snapPath, memPath) + }) + } + +} + func testCreateBalloon(ctx context.Context, t *testing.T, m *Machine) { if err := m.CreateBalloon(ctx, testBalloonMemory, testBalloonDeflateOnOom, testStatsPollingIntervals); err != nil { t.Errorf("Create balloon device failed from testAttachBalloon: %s", err) diff --git a/machineiface.go b/machineiface.go index 79e0a46e..a10a588c 100644 --- a/machineiface.go +++ b/machineiface.go @@ -23,7 +23,7 @@ var _ MachineIface = (*Machine)(nil) // MachineIface can be used for mocking and testing of the Machine. The Machine // is subject to change, meaning this interface would change. type MachineIface interface { - Start(context.Context) error + Start(context.Context, ...StartOpt) error StopVMM() error Shutdown(context.Context) error Wait(context.Context) error diff --git a/opts.go b/opts.go index 842772c0..cedde002 100644 --- a/opts.go +++ b/opts.go @@ -21,6 +21,7 @@ import ( // Opt represents a functional option to help modify functionality of a Machine. type Opt func(*Machine) +type StartOpt func(*Machine) // WithClient will use the client in place rather than the client constructed // during bootstrapping of the machine. This option is useful for mocking out @@ -47,3 +48,21 @@ func WithProcessRunner(cmd *exec.Cmd) Opt { machine.cmd = cmd } } + +// WithSnapshotOpt allows configuration of the snapshot config +// to be passed to LoadSnapshot +type WithSnapshotOpt func(*SnapshotConfig) + +// WithSnapshot will allow for the machine to start using a given snapshot. +func WithSnapshot(memFilePath, snapshotPath string, opts ...WithSnapshotOpt) StartOpt { + return func(m *Machine) { + m.Cfg.Snapshot.MemFilePath = memFilePath + m.Cfg.Snapshot.SnapshotPath = snapshotPath + + for _, opt := range opts { + opt(&m.Cfg.Snapshot) + } + + m.Handlers.FcInit = loadSnapshotHandlerList + } +} diff --git a/snapshot.go b/snapshot.go new file mode 100644 index 00000000..eefdc47b --- /dev/null +++ b/snapshot.go @@ -0,0 +1,21 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package firecracker + +type SnapshotConfig struct { + MemFilePath string + SnapshotPath string + EnableDiffSnapshots bool + ResumeVM bool +}