Skip to content

vmm: keep virtio activation alive in migration #105

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Coffeeri merged 1 commit into from
Mar 16, 2026
Merged

vmm: keep virtio activation alive in migration #105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 81 additions & 47 deletions vmm/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ use std::sync::mpsc::{
Receiver, RecvError, SendError, Sender, SyncSender, TryRecvError, TrySendError, channel,
sync_channel,
};
use std::sync::{Arc, Mutex};
use std::sync::{Arc, Mutex, Weak};
use std::thread::{JoinHandle, sleep};
#[cfg(not(target_arch = "riscv64"))]
use std::time::{Duration, Instant};
Expand Down Expand Up @@ -81,6 +81,7 @@ use crate::config::{RestoreConfig, add_to_config};
use crate::coredump::GuestDebuggable;
#[cfg(feature = "kvm")]
use crate::cpu::IS_IN_SHUTDOWN;
use crate::device_manager::DeviceManager;
use crate::landlock::Landlock;
use crate::memory_manager::MemoryManager;
use crate::migration::{get_vm_snapshot, recv_vm_config, recv_vm_state};
Expand Down Expand Up @@ -922,13 +923,40 @@ pub struct VmmThreadHandle {
pub http_api_handle: Option<HttpApiHandle>,
}

struct MigrationVmState {
// The migration worker owns the VM during migration, so this should stop
// working once that VM has been dropped.
device_manager: Weak<Mutex<DeviceManager>>,
}

impl MigrationVmState {
fn new(vm: &Vm) -> Self {
Self {
device_manager: Arc::downgrade(vm.device_manager()),
}
}

fn activate_virtio_devices(&self) -> result::Result<(), VmError> {
self.device_manager
.upgrade()
.expect("device manager should remain alive during migration")
.lock()
.unwrap()
.activate_virtio_devices()
.map_err(VmError::ActivateVirtioDevices)
}
}

/// Describes the current ownership of a running VM.
#[allow(clippy::large_enum_variant)]
pub enum MaybeVmOwnership {
enum MaybeVmOwnership {
/// The VMM holds the ownership of the VM.
Vmm(Vm),
/// The VM is temporarily blocked by the current ongoing migration.
Migration,
///
/// We still keep the device manager reachable so the epoll thread can
/// drain pending virtio activations while the migration worker owns the VM.
Migration(MigrationVmState),
/// No VM is running.
None,
}
Expand All @@ -939,13 +967,12 @@ impl MaybeVmOwnership {
/// # Panics
/// This method panics if `self` is not [`Self::Vmm`].
fn take_vm_for_migration(&mut self) -> Vm {
if !matches!(self, Self::Vmm(_)) {
panic!("should only be called when a migration can start");
}

match mem::replace(self, Self::Migration) {
MaybeVmOwnership::Vmm(vm) => vm,
_ => unreachable!(),
match mem::replace(self, Self::None) {
Self::Vmm(vm) => {
*self = Self::Migration(MigrationVmState::new(&vm));
vm
}
_ => panic!("should only be called when a migration can start"),
}
}

Expand Down Expand Up @@ -2942,7 +2969,7 @@ impl Vmm {
vm_config: Arc<Mutex<VmConfig>>,
prefault: bool,
) -> std::result::Result<(), VmError> {
if matches!(self.vm, MaybeVmOwnership::Migration) {
if matches!(self.vm, MaybeVmOwnership::Migration(_)) {
return Err(VmError::VmMigrating);
}

Expand Down Expand Up @@ -3175,7 +3202,7 @@ impl Vmm {
// Consume the event.
self.reset_evt.read().map_err(Error::EventFdRead)?;
// Workaround for guest-induced shutdown during a live-migration.
if matches!(self.vm, MaybeVmOwnership::Migration) {
if matches!(self.vm, MaybeVmOwnership::Migration(_)) {
self.postpone_lifecycle_event_during_migration(
PostMigrationLifecycleEvent::VmReboot,
);
Expand All @@ -3187,7 +3214,7 @@ impl Vmm {
info!("VM guest exit event");
self.guest_exit_evt.read().map_err(Error::EventFdRead)?;
// Workaround for guest-induced shutdown during a live-migration.
if matches!(self.vm, MaybeVmOwnership::Migration) {
if matches!(self.vm, MaybeVmOwnership::Migration(_)) {
self.postpone_lifecycle_event_during_migration(
PostMigrationLifecycleEvent::VmShutdown,
);
Expand All @@ -3201,11 +3228,18 @@ impl Vmm {
}
}
EpollDispatch::ActivateVirtioDevices => {
Comment thread
Coffeeri marked this conversation as resolved.
if let MaybeVmOwnership::Vmm(ref vm) = self.vm {
let count = self.activate_evt.read().map_err(Error::EventFdRead)?;
info!("Trying to activate pending virtio devices: count = {count}");
vm.activate_virtio_devices()
.map_err(Error::ActivateVirtioDevices)?;
let count = self.activate_evt.read().map_err(Error::EventFdRead)?;
info!("Trying to activate pending virtio devices: count = {count}");
match &self.vm {
MaybeVmOwnership::Vmm(vm) => vm
.activate_virtio_devices()
.map_err(Error::ActivateVirtioDevices)?,
MaybeVmOwnership::Migration(state) => {
state
.activate_virtio_devices()
.map_err(Error::ActivateVirtioDevices)?;
}
MaybeVmOwnership::None => {}
}
}
EpollDispatch::Api => {
Expand Down Expand Up @@ -3300,7 +3334,7 @@ impl RequestHandler for Vmm {
info!("Booting VM");
event!("vm", "booting");

if matches!(self.vm, MaybeVmOwnership::Migration) {
if matches!(self.vm, MaybeVmOwnership::Migration(_)) {
return Err(VmError::VmMigrating);
}

Expand Down Expand Up @@ -3376,15 +3410,15 @@ impl RequestHandler for Vmm {
fn vm_pause(&mut self) -> result::Result<(), VmError> {
match self.vm {
MaybeVmOwnership::Vmm(ref mut vm) => vm.pause().map_err(VmError::Pause),
MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?,
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating)?,
MaybeVmOwnership::None => Err(VmError::VmNotRunning)?,
}
}

fn vm_resume(&mut self) -> result::Result<(), VmError> {
match self.vm {
MaybeVmOwnership::Vmm(ref mut vm) => vm.resume().map_err(VmError::Resume),
MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?,
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating)?,
MaybeVmOwnership::None => Err(VmError::VmNotRunning)?,
}
}
Expand All @@ -3401,15 +3435,15 @@ impl RequestHandler for Vmm {
.map_err(VmError::SnapshotSend)
})
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?,
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating)?,
MaybeVmOwnership::None => Err(VmError::VmNotRunning)?,
}
}

fn vm_restore(&mut self, restore_cfg: RestoreConfig) -> result::Result<(), VmError> {
match &self.vm {
MaybeVmOwnership::Vmm(_vm) => return Err(VmError::VmAlreadyCreated),
MaybeVmOwnership::Migration => return Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => return Err(VmError::VmMigrating),
MaybeVmOwnership::None => (),
}

Expand Down Expand Up @@ -3464,15 +3498,15 @@ impl RequestHandler for Vmm {
MaybeVmOwnership::Vmm(ref mut vm) => {
vm.coredump(destination_url).map_err(VmError::Coredump)
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => Err(VmError::VmNotRunning),
}
}

fn vm_shutdown(&mut self) -> result::Result<(), VmError> {
let vm = match self.vm {
MaybeVmOwnership::Vmm(ref mut vm) => vm,
MaybeVmOwnership::Migration => return Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => return Err(VmError::VmMigrating),
MaybeVmOwnership::None => return Err(VmError::VmNotRunning),
};
// Drain console_info so that the FDs are not reused
Expand All @@ -3493,7 +3527,7 @@ impl RequestHandler for Vmm {
// First we stop the current VM
let vm = match self.vm {
MaybeVmOwnership::Vmm(ref mut vm) => vm,
MaybeVmOwnership::Migration => return Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => return Err(VmError::VmMigrating),
MaybeVmOwnership::None => return Err(VmError::VmNotRunning),
};
let config = vm.get_config();
Expand Down Expand Up @@ -3566,7 +3600,7 @@ impl RequestHandler for Vmm {
let state = match &self.vm {
MaybeVmOwnership::Vmm(vm) => vm.get_state(),
// TODO in theory one could live-migrate a non-running VM ..
MaybeVmOwnership::Migration => VmState::Running,
MaybeVmOwnership::Migration(_) => VmState::Running,
MaybeVmOwnership::None => VmState::Created,
};

Expand All @@ -3575,14 +3609,14 @@ impl RequestHandler for Vmm {
MaybeVmOwnership::Vmm(vm) => {
memory_actual_size -= vm.balloon_size();
}
MaybeVmOwnership::Migration => {}
MaybeVmOwnership::Migration(_) => {}
MaybeVmOwnership::None => {}
}

let device_tree = match &self.vm {
MaybeVmOwnership::Vmm(vm) => Some(vm.device_tree().lock().unwrap().clone()),
// TODO we need to fix this
MaybeVmOwnership::Migration => None,
MaybeVmOwnership::Migration(_) => None,
MaybeVmOwnership::None => None,
};

Expand Down Expand Up @@ -3624,7 +3658,7 @@ impl RequestHandler for Vmm {
MaybeVmOwnership::None => {
self.vm_config = None;
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating)?,
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating)?,
}

Ok(())
Expand Down Expand Up @@ -3654,7 +3688,7 @@ impl RequestHandler for Vmm {
.inspect_err(|e| error!("Error when resizing VM: {e:?}"))?;
Ok(())
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => {
let mut config = self.vm_config.as_ref().unwrap().lock().unwrap();
if let Some(desired_vcpus) = desired_vcpus {
Expand Down Expand Up @@ -3687,7 +3721,7 @@ impl RequestHandler for Vmm {
Ok(())
}
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => Err(VmError::ResizeDisk),
}
}
Expand All @@ -3701,7 +3735,7 @@ impl RequestHandler for Vmm {
.inspect_err(|e| error!("Error when resizing zone: {e:?}"))?;
Ok(())
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => {
// Update VmConfig by setting the new desired ram.
let memory_config = &mut self.vm_config.as_ref().unwrap().lock().unwrap().memory;
Expand Down Expand Up @@ -3743,7 +3777,7 @@ impl RequestHandler for Vmm {
.map(Some)
.map_err(VmError::SerializeJson)
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => {
// Update VmConfig by adding the new device.
let mut config = self.vm_config.as_ref().unwrap().lock().unwrap();
Expand Down Expand Up @@ -3775,7 +3809,7 @@ impl RequestHandler for Vmm {
.map(Some)
.map_err(VmError::SerializeJson)
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => {
// Update VmConfig by adding the new device.
let mut config = self.vm_config.as_ref().unwrap().lock().unwrap();
Expand All @@ -3792,7 +3826,7 @@ impl RequestHandler for Vmm {
.inspect_err(|e| error!("Error when removing device from the VM: {e:?}"))?;
Ok(())
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => {
if let Some(ref config) = self.vm_config {
let mut config = config.lock().unwrap();
Expand Down Expand Up @@ -3827,7 +3861,7 @@ impl RequestHandler for Vmm {
.map(Some)
.map_err(VmError::SerializeJson)
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => {
// Update VmConfig by adding the new device.
let mut config = self.vm_config.as_ref().unwrap().lock().unwrap();
Expand Down Expand Up @@ -3856,7 +3890,7 @@ impl RequestHandler for Vmm {
.map(Some)
.map_err(VmError::SerializeJson)
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => {
// Update VmConfig by adding the new device.
let mut config = self.vm_config.as_ref().unwrap().lock().unwrap();
Expand Down Expand Up @@ -3885,7 +3919,7 @@ impl RequestHandler for Vmm {
.map(Some)
.map_err(VmError::SerializeJson)
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => {
// Update VmConfig by adding the new device.
let mut config = self.vm_config.as_ref().unwrap().lock().unwrap();
Expand Down Expand Up @@ -3914,7 +3948,7 @@ impl RequestHandler for Vmm {
.map(Some)
.map_err(VmError::SerializeJson)
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => {
// Update VmConfig by adding the new device.
let mut config = self.vm_config.as_ref().unwrap().lock().unwrap();
Expand Down Expand Up @@ -3943,7 +3977,7 @@ impl RequestHandler for Vmm {
.map(Some)
.map_err(VmError::SerializeJson)
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => {
// Update VmConfig by adding the new device.
let mut config = self.vm_config.as_ref().unwrap().lock().unwrap();
Expand Down Expand Up @@ -3977,7 +4011,7 @@ impl RequestHandler for Vmm {
.map(Some)
.map_err(VmError::SerializeJson)
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => {
// Update VmConfig by adding the new device.
let mut config = self.vm_config.as_ref().unwrap().lock().unwrap();
Expand All @@ -3997,23 +4031,23 @@ impl RequestHandler for Vmm {
.map(Some)
.map_err(VmError::SerializeJson)
}
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => Err(VmError::VmNotRunning),
}
}

fn vm_power_button(&mut self) -> result::Result<(), VmError> {
match self.vm {
MaybeVmOwnership::Vmm(ref mut vm) => vm.power_button(),
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => Err(VmError::VmNotRunning),
}
}

fn vm_nmi(&mut self) -> result::Result<(), VmError> {
match self.vm {
MaybeVmOwnership::Vmm(ref mut vm) => vm.nmi(),
MaybeVmOwnership::Migration => Err(VmError::VmMigrating),
MaybeVmOwnership::Migration(_) => Err(VmError::VmMigrating),
MaybeVmOwnership::None => Err(VmError::VmNotRunning),
}
}
Expand Down Expand Up @@ -4103,7 +4137,7 @@ impl RequestHandler for Vmm {
) -> result::Result<(), MigratableError> {
match self.vm {
MaybeVmOwnership::Vmm(_) => (),
MaybeVmOwnership::Migration => {
MaybeVmOwnership::Migration(_) => {
return Err(MigratableError::MigrateSend(anyhow!(
"There is already an ongoing migration"
)));
Expand Down Expand Up @@ -4192,7 +4226,7 @@ impl RequestHandler for Vmm {

fn vm_cancel_migration(&mut self) -> result::Result<(), MigratableError> {
match self.vm {
MaybeVmOwnership::Migration => (),
MaybeVmOwnership::Migration(_) => (),
_ => {
return Err(MigratableError::CancelMigration(anyhow!(
"There is no ongoing migration"
Expand Down
Loading
Loading