kvmrun

#!/bin/bash
#
# Known issues:
# * qemu-system-x86_64 doesn't support l2-cache-size so cluster_size set to 256K
# * q35 and pc machine types have different PCI bus layout so ethernet/cdrom/boot
#   will be involved.
# * can not using "-cpu host,kvm=off" option because Property '.kvm' not found
#   You need at least 2.1 for the kvm=off property.
# * global variable can not be changed inside subshell access, ie, $(func arg)
#   https://stackoverflow.com/questions/23564995/how-to-modify-a-global-variable-within-a-function-in-bash
# * USB pass-through requires root privilege.
#   https://serverfault.com/questions/666210/attaching-usb-dongle-to-kvm-vm
# * QEMU 2.0 doesn't support USB3.0 so do not use "-device qemu-xhci"
#   https://github.com/qemu/qemu/blob/master/docs/usb2.txt
# * Currently, you need to run qemu as root to use tun/tap networking 
#   https://wiki.qemu.org/Documentation/Networking/NAT
#   https://wiki.qemu.org/Documentation/Networking
# * Windows XP doesn't support Q35 machine type
# * DO NOT load snapshots automatically, which may make serious damage to your work.
#   The snapshots also save the contents of the filesystem. When loading an old
#   snapshot, the recent changes in your filesystem will be totally wiped out.
# * Debian 8 seems not pretty well to handle Intel(R) Core(TM) i7-7500U as the host.
#   It caused loads of "NMI received for unknown reason 21/31 on CPU 0" events at
#   every terminals. Using "-cpu kvm64" can workaround.
#   https://www.spinics.net/lists/kvm/msg94595.html
#

# debug only
#RUN=echo
RUN=eval
#QEMU=qemu-system-x86_64
QEMU=/usr/libexec/qemu-kvm

#############################################################################
# Configure
#############################################################################
CFG_ISO=
CFG_CPU=1
CFG_RAM=2G
CFG_HDS=16G
CFG_SCREEN=spice     	# sdl/spice/gtk
CFG_VIRTIO=yes
CFG_USBS=               # usb storage to ehci bus
CFG_USBP=               # "BUS,ADDR" pair for usb pass-through
CFG_NETWORK=user        # tap,user,
CFG_PROFILE=            # winxp, win7

#############################################################################
# System definition
#############################################################################
# KVM Machine default configurations
VMSPICEPORT=5930
VMDRVID=0
VMHDD=""
VMNICMODL="rtl8139"	# ne2k_pci,i82551,i82557b,i82559er,rtl8139,e1000,pcnet

#VMVIDEO="-device qxl-vga,id=video0,ram_size=67108864,vram_size=67108864,\
#          vram64_size_mb=0,vgamem_mb=16,bus=pci.0,addr=0x2"
VMVIDEO="-vga qxl"
#VMAUDIO="-soundhw hda"

#VMIOMMU="-enable-kvm -machine q35,accel=kvm"
VMIOMMU="-enable-kvm -machine pc,accel=kvm"

VMSPICE="-device virtio-serial-pci \
	-device virtserialport,chardev=spicechannel0,name=com.redhat.spice.0 \
	-chardev spicevmc,id=spicechannel0,name=vdagent"

#VMFLAGS="$VMIOMMU $VMVIDEO $VMAUDIO"


#############################################################################
# System Support Group
#############################################################################
System()
{
  if test -e /etc/centos-release; then
    echo "CENTOS"
  elif test -e /etc/debian_version; then
    grep Ubuntu /etc/os-release > /dev/null
    if test "x$?" = "x0"; then
      echo "UBUNTU"
    else
      echo "DEBIAN"
    fi
  elif test -e /etc/arch-release; then
    echo "ARCHLINUX"
  else
    echo "UNKNOWN"
  fi
}


# https://stackoverflow.com/questions/4023830/how-to-compare-two-strings-in-dot-separated-version-format-in-bash
# Usage:
# 1) if test $(Qemu_Version_match 2.0.1); then echo TRUE; else echo FALSE; fi
# 2) Qemu_Version_match 2.0.1 > /dev/null
#    echo $?
# 20180910: Debian version info: QEMU emulator version 2.0.0(Debian ...)
Qemu_Version_match()
{
  local CURR=$($QEMU --version 2> /dev/null | grep version | tr ",.)(" " " | cut -f4-6 -d' ')
  CURR=$(printf "%d%03d%03d" $CURR)

  local WANT="0"		# if target version not specified
  if test "x$1" != "x"; then
    WANT=$(printf "%d%03d%03d" $(echo $1 | tr '.' ' '))
  fi
  echo "$CURR -ge $WANT"
  return $(expr $CURR \>= $WANT)
}

# FIXME: qemu-system-x86_64 doesn't support l2-cache-size
# qemu-system-x86_64: -drive file=vdisk_hda.kvm,l2-cache-size=2M: 
#   could not open disk image vdisk_hda.kvm: Block format 'qcow2' used by 
#   device 'ide0-hd0' doesn't support the option 'l2-cache-size'
# FIXME: qemu-system-x86_64 doesn't support cache=none in an empty qcow2 image.
# qemu-system-x86_64: -drive file=kkk_hda.kvm,if=none,id=qcow2,aio=native,cache=none: \
#   could not open disk image kkk_hda.kvm: Could not read L1 table: Invalid argument
# Note: Generally, do not use aio=native option along with cache=none for QEMU. 
#   https://access.redhat.com/articles/41313
# qemu: use "ide" as device name for implicit SATA controller on Q35
#   https://www.redhat.com/archives/libvir-list/2013-September/msg01198.html
# 
# qemu [...] -drive if=none,id=drive0,format=raw -device virtio-blk-pci,drive=drive0,scsi=off ...
add_drive()
{
  local TYPE=$(qemu-img info "$1" | grep 'file format:' | cut -d' ' -f3)
  #local SIZE=$(qemu-img info "$1" | grep 'virtual size:' | cut -d'(' -f2 | cut -d' ' -f1)
  local SIZE=0
  local OPT_CACHE=""
  #local OPT_AIO=",aio=native,cache=none"

  case "$TYPE" in
    qcow2)
      if [ $SIZE -ge 68719476736 ]; then
        OPT_CACHE=",l2-cache-size=8M"
      elif [ $SIZE -ge 34359738368 ]; then
        OPT_CACHE=",l2-cache-size=4M"
      elif [ $SIZE -ge 17179869184 ]; then
        OPT_CACHE=",l2-cache-size=2M"
      fi ;;

    raw|vvfat|vpc|vmdk|vhdx|vdi)
      ;;
        
    *) return ;;
  esac

  if test "x$CFG_VIRTIO" = "xyes"; then
    echo "-drive if=none${OPT_AIO},id=ide${VMDRVID},file=\"${1}\",format=$TYPE${OPT_CACHE} \
	  -device virtio-blk-pci,scsi=off,drive=ide${VMDRVID}"
  else
    #echo "-drive if=none${OPT_AIO},id=ide${VMDRVID},file=\"${1}\",format=$TYPE${OPT_CACHE} \
    #	  -device ide-hd,bus=ide.2,drive=ide${VMDRVID}"
    echo "-drive if=none${OPT_AIO},id=ide${VMDRVID},file=\"${1}\",format=$TYPE${OPT_CACHE} \
	  -device ide-hd,drive=ide${VMDRVID}"
  fi
} 

vm_cdrom()
{
  if test ! -e "$1"; then
    return
  fi

  if test "$(qemu-img info "$1" | grep 'file format:' | cut -d' ' -f3)" != "raw"; then
    return
  fi

  #VMHDD="$VMHDD -drive if=none,format=raw,snapshot=on,id=cdrom0,file=\"${1}\" \
  #	  -device ide-cd,bus=ide.1,drive=cdrom0 -boot d"
  VMHDD="$VMHDD -drive if=none,format=raw,snapshot=on,id=cdrom0,file=\"${1}\" \
	  -device ide-cd,drive=cdrom0 -boot d"
}


# searching the available vm file systems
# $1: filename or vm name
# $2: output string
vm_search()
{

  local TMPDRV=

  VMDISK=""
  if test -e "$1"; then
    # if explicily given the vm file system
    TMPDRV=$(add_drive "$1")
    VMHDD="$VMHDD $TMPDRV"
    VMDRVID=$(expr $VMDRVID + 1)
    VMDISK="$1"
    qemu-img info "$1" >> /tmp/kvmrun.log
  else
    # if given the vm name then searching for it's file systems 
    # in the name pattern: xxx_hdx.*
    for i in ${1}_hd?.{qcow2,raw,vvfat,vpc,vmdk,vhdx,vdi}
    do
      if test -e "$i"; then
        TMPDRV=$(add_drive "$i")
	VMHDD="$VMHDD $TMPDRV"
	VMDRVID=$(expr $VMDRVID + 1)
	VMDISK="$i"
	qemu-img info "$i" >> /tmp/kvmrun.log
      fi
    done
  fi
  #echo VS: $VMHDD
}


# FIXME: random may not be a good idea. Fix the MAC to specified VM looks better
#    printf "52:54:%02x:%02x:%02x:%02x" $(( $RANDOM & 0xff)) $(( $RANDOM & 0xff )) \
#        $(( $RANDOM & 0xff)) $(( $RANDOM & 0xff ))
# FIXME: -device virtio-net-pci performance better than -net nic,model=virtio
#     echo "-net nic,model=virtio,macaddr=${macaddr}"
#     https://www.linux-kvm.org/page/10G_NIC_performance:_VFIO_vs_virtio
# FIXME: could not configure /dev/net/tun (tap0): Operation not permitted
#     -netdev tap,id=net0,ifname=tap0,script=/etc/qemu-kvm/qemu-ifup \
#             -device virtio-net-pci,netdev=net0,mac=${macaddr}"
add_nic()
{

  local nhash=$(echo "$1" | md5sum)
  local macaddr="52:54:${nhash:0:2}:${nhash:2:2}:${nhash:4:2}:${nhash:6:2}"
  local mynet=""

  if test "x$CFG_NETWORK" = "xtap"; then
    mynet="$mynet -netdev tap,id=net0,ifname=tap0,script=no,downscript=no,vhost=on"
  else
    mynet="$mynet -netdev user,id=net0"
  fi

  if test "x$CFG_VIRTIO" = "xyes"; then
    mynet="$mynet -device virtio-net-pci,netdev=net0,mac=${macaddr}"
  else
    mynet="$mynet -device $VMNICMODL,netdev=net0,mac=${macaddr}"
  fi
  echo $mynet
}


# $1: size  $2: name
vm_create_disk()
{
  if test ! -e "$2"; then
    echo qemu-img create -f qcow2 -o cluster_size=256K "$2" "$1"
    $RUN qemu-img create -f qcow2 -o cluster_size=256K \"$2\" \"$1\"
  fi
}

list_iommu()
{
    shopt -s nullglob
    for d in /sys/kernel/iommu_groups/*/devices/*; do
        n=${d#*/iommu_groups/*}; n=${n%%/*}
        printf 'IOMMU Group %s ' "$n"
        lspci -nns "${d##*/}"
    done
}


# $1: [-s|-d]  $2: disk image
vmdisk_list()
{
  if test -e "$2"; then
    if test "x$1" = "x-s"; then
      qemu-img snapshot -l "$2"
    else
      qemu-img info "$2"
    fi
  else
    for i in ${2}_hd?.{qcow2,raw,vvfat,vpc,vmdk,vhdx,vdi}
    do
      if test -e "$i"; then
        if test "x$1" = "x-s"; then
          qemu-img snapshot -l "$i"
	else
	  qemu-img info "$i"
	  echo ""
	fi
      fi
    done
  fi
}

# $1: disk image
# https://en.wikibooks.org/wiki/QEMU
# assuming the running kernel support auto max partition in loop device.
# assuming nbd module is available with the running kernel
# assuming there's no lvm partition
vmdisk_mount()
{
  case "$1" in
    *.raw)
      losetup -f -P "$1"
      if test "x$?" != "x0"; then
        return
      fi
      for i in /dev/loop0p*
      do
        mkdir /media/$(basename "$i")
	mount "$i" /media/$(basename "$i")
      done
      ;;
    *.qcow2)
      modprobe nbd max_part=16
      qemu-nbd -c /dev/nbd0 "$1"
      partprobe -s /dev/nbd0
      for i in /dev/nbd0p*
      do
        mkdir /media/$(basename "$i")
	mount "$i" /media/$(basename "$i")
      done
      ;;
    *) echo Unrecognized disk image.
      ;;
  esac
}

# $1: disk image
vmdisk_umount()
{
  case "$1" in
    *.raw)
      for i in /dev/loop0p*
      do
        if test -e "/media/$(basename $i)"; then
	  umount /media/$(basename $i)
	  rmdir /media/$(basename $i)
	fi
      done
      losetup -D
      ;;
    *.qcow2)
      for i in /dev/nbd0p*
      do
        if test -e "/media/$(basename $i)"; then
	  umount /media/$(basename $i)
	  rmdir /media/$(basename $i)
	fi
      done
      qemu-nbd -d /dev/nbd0
      killall qemu-nbd
      ;;
    *) echo Unrecognized disk image.
      ;;
  esac
}

vm_usb_device_on()
{
  if test "x$1" = "x"; then
    local BUS=$(echo $CFG_USBP | cut -f1 -d,)
    local ADDR=$(echo $CFG_USBP | cut -f2 -d,)
  else
    local BUS=$(echo $1 | cut -f1 -d,)
    local ADDR=$(echo $1 | cut -f2 -d,)
  fi

  local USBDEV=$(printf "/dev/bus/usb/%03d/%03d" $BUS $ADDR)
  local ID=$(id -nu)
  
  sudo chown $ID $USBDEV
  if test "x$1" = "x"; then
    echo "-device usb-host,hostbus=$BUS,hostaddr=$ADDR"
  else
    cat << USBON
Device $USBDEV permission acquired:
    $(ls -l $USBDEV)
You may use (obseleted):
    usb_add host:$BUS.$ADDR
    usb_del $BUS.$ADDR
Or:
    device_add usb-host,id=ANYID,hostbus=$BUS,hostaddr=$ADDR
    device_del ANYID
In the QEMU monitor to plug and unplug the device. Helpful commands also:
    info usbhost
    info usb

USBON
  fi
}

vm_usb_device_off()
{
  if test "x$1" = "x"; then
    local BUS=$(echo $CFG_USBP | cut -f1 -d,)
    local ADDR=$(echo $CFG_USBP | cut -f2 -d,)
  else
    local BUS=$(echo $1 | cut -f1 -d,)
    local ADDR=$(echo $1 | cut -f2 -d,)
  fi
  local USBDEV=$(printf "/dev/bus/usb/%03d/%03d" $BUS $ADDR)

  echo "Recover the ownership of $USBDEV"
  sudo chown root $USBDEV
  ls -l $USBDEV
}


# see https://gist.github.com/extremecoders-re/e8fd8a67a515fee0c873dcafc81d811c
# bridge-utils iptables dnsmasq tunctl must be installed priorily
host_network_bridge_on()
{
  local NIC=$(ls -1 /sys/class/net | grep ^en)

  if test "x$1" != "x"; then
    ls -d /sys/class/net/$1 > /dev/null 2> /dev/null
    if test "x$?" = "x0"; then
      NIC=$1
    else
      echo network interface [$1] not found!
      return
    fi
  fi

  if test -e /sys/class/net/br0; then
    echo Bridge device 'br0' already exists.
    return
  fi

  # this function requires sudo/root priviledge
  if test $(id -u) != "0"; then
    echo root priviledge required.
    return
  fi

  # Create a bridge
  brctl addbr br0

  # Clear IP of ethernet port
  ip addr flush dev $NIC

  # Add ethernet port to bridge
  brctl addif br0 $NIC

  # Create tap interface
  # FIXME: root???
  tunctl -t tap0 -u root

  # Add tap0 to bridge
  brctl addif br0 tap0

  # Make sure everything is up
  ifconfig $NIC up
  ifconfig tap0 up
  ifconfig br0 up

  # Check if properly bridged
  brctl show

  # Assign ip to br0
  dhclient -v br0
}

host_network_bridge_off()
{
  local NIC=$(ls -1 /sys/class/net | grep ^en)

  if test "x$1" != "x"; then
    ls -d /sys/class/net/$1 > /dev/null 2> /dev/null
    if test "x$?" = "x0"; then
      NIC=$1
    else
      echo network interface [$1] not found!
      return
    fi
  fi

  if test ! -e /sys/class/net/br0; then
    echo Bridge device 'br0' does not exist.
    return
  fi


  # this function requires sudo/root priviledge
  if test $(id -u) != "0"; then
    echo root priviledge required.
    return
  fi

  # Remove tap interface tap0 from bridge br0
  brctl delif br0 tap0

  # Delete tap0
  tunctl -d tap0

  # Remove ethernet port from bridge
  brctl delif br0 $NIC

  # Bring bridge down
  ifconfig br0 down

  # Remove bridge
  brctl delbr br0

  # Bring ethernet port up
  ifconfig $NIC up

  # Check if an IP is assigned to ethernet port, if not request one
  dhclient -v $NIC
}

usage_exit()
{
    cat << my_usage
$0 [OPTION] KVM_IMAGE_NAME
OPTION:
  -c, --cpu=NUM              Number of allocated CPU
  -d, --display=DISPLAY      Set display to [gtk/sdl/spice]
  -i, --install=IMAGE        The installer ISO image
  -p, --profile=PROFILE      Specify the VM profile
  -q  [pc,q35]               Specify the machine type
  -r, --ram=RAM              Define the size of RAM [2G]
  -s, --hdd-size=SIZE        Define the size of hard disks [16G]
  -u, --usb-pass=BUS,ADDR    Request USB device pass through (sudo)
      --usb-disk IMAGE       Insert a USB storage disk
      --novirtio             Disable the virtio devices
      --tap                  Enable the tap device networking

$0 [COMMAND] [KVM_IMAGE_NAME]
COMMAND:
      --usb-add BUS,ADDR     Add the USB device for hotplug (sudo)
      --usb-del BUS,ADDR     Release the USB device (sudo)
      --tapbridge on/off     turn on or off the network bridge in host machine
      --iommu                List the IOMMU Group
      --snapshot             List the known snapshots
      --vmdisk               List the VM disk volumn
      --mount                Mount the VM disk image
      --umount               Umount the VM disk image
      --vnc [port]           Reconnect the remote viewer

KVM_IMAGE_NAME is the base name of the KVM virtual machine.
The correspondent image is KVM_IMAGE_NAME_hda.kvm.

my_usage
    exit 0
}

#############################################################################
# main
#############################################################################
while [ "$#" -gt 0 ]; do 
  case "$1" in
    -h|--help) usage_exit;;

    -c) CFG_CPU="$2"; shift;;
    --cpu=*) CFG_CPU="${1#*=}";;

    -d) CFG_SCREEN="$2"; shift;;
    --display) CFG_SCREEN="${1#*=}";;

    -i) CFG_ISO="$2"; shift;;
    --install=*) CFG_ISO="${1#*=}";;

    -p) CFG_PROFILE="$2"; shift;;
    --profile=*) CFG_PROFILE="${1#*=}";;

    -q) 
      if test "x$2" = "xpc"; then
        VMIOMMU="-enable-kvm -machine pc,accel=kvm"
      elif test "x$2" = "xq35"; then
        VMIOMMU="-enable-kvm -machine q35,accel=kvm"
      fi
      shift;;

    -r) CFG_RAM="$2"; shift;;
    --ram=*) CFG_RAM="${1#*=}";;

    -s) CFG_HDS="$2"; shift;;
    --hdd-size=*) CFG_HDS="${1#*=}";;

    -u) CFG_USBP="$2"; shift;;
    --usb-pass=*) CFG_USBP="${1#*=}";;

    --usb-disk)
      if test "x$2" = "x" -o "x${2:0:1}" = "x-"; then
        cat << USBDISK
Usage: --usb-disk usb-disk-image
You may use the following command in the QEMU monitor later to access the USB storage
  drive_add 0 if=none,id=usbdisk1,file=usb-disk-image
  device_add usb-storage,bus=ehci.0,drive=usbdisk1 
USBDISK
	exit 0
      elif test -e "$2"; then
        CFG_USBS="$2"
      fi
      shift;;

    --novirtio) CFG_VIRTIO="no" ;;
    --tap) CFG_NETWORK="tap" ;;

    ######################################################################
    # command section
    ######################################################################
    --usb-add) vm_usb_device_on  $2; exit 0;;
    --usb-del) vm_usb_device_off $2; exit 0;;

    --tapbridge)
      if test "x$2" = "xon"; then
        host_network_bridge_on $3
      else
        host_network_bridge_off $3
      fi
      exit 0;;

    --iommu) list_iommu; exit 0;;

    --snapshot) vmdisk_list -s "$2"; exit 0;;
    --vmdisk)   vmdisk_list -d "$2"; exit 0;;

    --mount) vmdisk_mount "$2"; exit 0;;
    --umount) vmdisk_umount "$2"; exit 0;;

    --vnc) 
      if test "x$2" = "x"; then
	ps auxw | grep qemu-system
      else
	remote-viewer spice://127.0.0.1:$2 &
      fi
      exit 0;;

    ### TEST and DEBUG section
    --ver-cmp)
        if test $(Qemu_Version_match $2); then echo TRUE; else echo false; fi
	Qemu_Version_match $2; echo $?
	exit 0;;

    -*) echo Unknown parameter [$@]; exit 1;;
    *) break;;
  esac
  shift
done

if test "x$CFG_PROFILE" = "x"; then
  # the simplest way to find if the client is a windows xp box
  echo $1 | grep [Ww]in.*[xX][pP] > /dev/null
  if test "x$?" = "x0"; then
    CFG_PROFILE="winxp"
  else
    echo $1 | grep [Ww]in.*7 > /dev/null
    if test "x$?" = "x0"; then
      CFG_PROFILE="win7"
    fi
  fi
fi

if [ "$#" = "0" ]; then 
  usage_exit
fi

export QEMU_AUDIO_DRV=pa
echo "$(date) QEMU running on $1" >> /tmp/kvmrun.log

# Run the vmachine
if test ! -e "$CFG_ISO"; then		# no installation is required
  vm_search "$1"
else
  vm_cdrom "$CFG_ISO"
  if test "x$VMHDD" = "x"; then
    echo "$CFG_ISO: not an ISO image!"
  else
    vm_search "$1"
    if test "x$VMDISK" = "x"; then
      echo "Creating VM [$1] as qcow2, $CFG_HDS"
      vm_create_disk $CFG_HDS ${1}_hda.qcow2
      vm_search ${1}_hda.qcow2
    fi

    # added an extra block devices while installing, for Windows client.
    # because Windows could not recognize virtio driver so it will start
    # in non-virtio mode but include a dummy driver in virtio block device
    # so give Windows an opportunate to search the driver.
    case "$CFG_PROFILE" in
      win*)
        vm_create_disk $CFG_HDS dummy.qcow2
        VMHDD="$VMHDD -drive if=none,id=idexp,file=\"dummy.qcow2\",format=qcow2 \
 	            -device virtio-blk-pci,scsi=off,drive=idexp"
	;;
    esac
  fi
fi

if test "x$VMHDD" = "x"; then
  echo KVM Image [$1] not found
  exit 1
fi

# extra block devices
#VMHDD="-drive if=none,format=raw,snapshot=on,id=cdrom1,file=\"../Downloads/vboximg/virtio-win-0.1.141.iso\" \
#		-device ide-cd,drive=cdrom1 $VMHDD"
#VMHDD="$VMHDD -drive if=none,id=idex0,file=\"reactos_hda.qcow2\",format=qcow2 \
#		-device virtio-blk-pci,scsi=off,drive=idex0"

# Debian 8 seems not well to handle Intel(R) Core(TM) i7-7500U as the host
#VMRTIME="-name $1 -m $CFG_RAM -cpu host"
VMRTIME="-name $1 -m $CFG_RAM"
case "$CFG_CPU" in
  *k) VMRTIME="$VMRTIME -cpu kvm32" ;;
  *K) VMRTIME="$VMRTIME -cpu kvm64" ;;
  *)  VMRTIME="$VMRTIME -cpu host" ;;
esac
case "$CFG_CPU" in
  2*) VMRTIME="$VMRTIME -smp 2,sockets=1,cores=1,threads=2" ;;
  4*) VMRTIME="$VMRTIME -smp 4,sockets=2,cores=2,threads=1" ;;
  8*) VMRTIME="$VMRTIME -smp 4,sockets=2,cores=2,threads=2" ;;
esac


if test "x$CFG_PROFILE" = "xwinxp"; then
  VMRTIME="$VMRTIME -enable-kvm -machine pc,accel=kvm $VMVIDEO -soundhw ac97"
else
  VMRTIME="$VMRTIME $VMIOMMU $VMVIDEO $VMAUDIO"

  # Don't use that if virtio is not specified, especially in some old client like 
  # Windows XP/2000, where no RNG device or virtio pci bus were supported.
  if test "x$CFG_VIRTIO" = "xyes"; then
    VMRTIME="$VMRTIME -object rng-random,filename=/dev/random,id=rng0 -device virtio-rng-pci,rng=rng0"
  fi
fi

# USB3.0 requires QEMU 2.1.x
# Otherwise rollback to USB2.0 and put usb-tablet in the bus of USB1.1
# FIXME: Debian 9/QEMU 2.9.x still missing qemu-xhci
if test $(Qemu_Version_match 2.1.0); then
  #VMRTIME="$VMRTIME -device qemu-xhci -device usb-tablet"
  VMRTIME="$VMRTIME -usb -device usb-ehci,id=ehci -device usb-tablet,bus=usb-bus.0"
else
  VMRTIME="$VMRTIME -usb -device usb-ehci,id=ehci -device usb-tablet,bus=usb-bus.0"
fi
if test "x$CFG_USBS" != "x"; then
  VMRTIME="$VMRTIME -drive if=none,id=usbkey1,file=$CFG_USBS"
  VMRTIME="$VMRTIME -device usb-storage,bus=ehci.0,id=usbkey1,drive=usbkey1"
fi
if test "x$CFG_USBP" != "x"; then
  VMRTIME="$VMRTIME $(vm_usb_device_on)"
fi

VMRTIME="$VMRTIME $(add_nic ${1}) $VMHDD"

case "$CFG_SCREEN" in
  sdl) VMRTIME="$VMRTIME -display sdl,window_close=off -daemonize" 
    #qemu-img snapshot -l "$VMDISK" | grep sdl > /dev/null
    #if test "x$?" = "x0"; then
    #  VMRTIME="$VMRTIME -loadvm sdl"
    #fi
    echo "Switch to monitor window by ctrl-alt-2"
    ;;
  gtk) VMRTIME="$VMRTIME -display gtk -no-quit -daemonize" 
    #qemu-img snapshot -l "$VMDISK" | grep gtk > /dev/null
    #if test "x$?" = "x0"; then
    #  VMRTIME="$VMRTIME -loadvm gtk"
    #fi
    ;;
  spice|*) 
    Instance=$(ps -e | grep qemu-system-x86 | wc -l)
    VMSPICEPORT=$(expr $VMSPICEPORT + $Instance)
    #https://unix.stackexchange.com/questions/426652/connect-to-running-qemu-instance-with-qemu-monitor
    #VMONITOR="-monitor stdio"
    VMONITOR="-monitor tcp:127.0.0.1:1$VMSPICEPORT,server,nowait -daemonize"
    VMRTIME="$VMRTIME $VMSPICE -spice port=$VMSPICEPORT,disable-ticketing $VMONITOR"
    #qemu-img snapshot -l "$VMDISK" | grep spice > /dev/null
    #if test "x$?" = "x0"; then
    #  VMRTIME="$VMRTIME -loadvm spice"
    #fi
    echo "remote-viewer spice://127.0.0.1:$VMSPICEPORT &"
    remote-viewer spice://127.0.0.1:$VMSPICEPORT&
    ;;
esac

echo $QEMU $VMRTIME | tee -a /tmp/kvmrun.log

# tap/bridge networking require sudo/root priviledge
if test "$CFG_NETWORK" = "tap"; then
  $RUN sudo $QEMU $VMRTIME
else
  $RUN $QEMU $VMRTIME
fi