Skip to content

Commit

Permalink
feat: add support for io.bus and io.cache for shared filesystems with…
Browse files Browse the repository at this point in the history 
… VMs

Signed-off-by: Ruihua Wen <spiffyeight77@gmail.com>
  • Loading branch information
@SpiffyEight77
SpiffyEight77 committed Jun 21, 2024
1 parent 2bbcd64 commit 5cc1e00
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 56 deletions.
13 changes: 11 additions & 2 deletions internal/server/device/device_utils_disk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -426,7 +426,7 @@ func DiskVMVirtfsProxyStop(pidPath string) error {
// Returns UnsupportedError error if the host system or instance does not support virtiosfd, returns normal error
// type if process cannot be started for other reasons.
// Returns revert function and listener file handle on success.
func DiskVMVirtiofsdStart(execPath string, inst instance.Instance, socketPath string, pidPath string, logPath string, sharePath string, idmaps []idmap.Entry) (func(), net.Listener, error) {
func DiskVMVirtiofsdStart(execPath string, inst instance.Instance, socketPath string, pidPath string, logPath string, sharePath string, idmaps []idmap.Entry, cacheOption string) (func(), net.Listener, error) {
revert := revert.New()
defer revert.Fail()

Expand Down Expand Up @@ -499,8 +499,17 @@ func DiskVMVirtiofsdStart(execPath string, inst instance.Instance, socketPath st

defer func() { _ = unixFile.Close() }()

switch cacheOption {
case "metadata":
cacheOption = "metadata"
case "unsafe":
cacheOption = "always"
default:
cacheOption = "never"
}

// Start the virtiofsd process in non-daemon mode.
args := []string{"--fd=3", "--cache=never", "-o", fmt.Sprintf("source=%s", sharePath)}
args := []string{"--fd=3", fmt.Sprintf("--cache=%s", cacheOption), "-o", fmt.Sprintf("source=%s", sharePath)}
proc, err := subprocess.NewProcess(cmd, args, logPath, logPath)
if err != nil {
return nil, nil, err
Expand Down
120 changes: 66 additions & 54 deletions internal/server/device/disk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,8 +334,8 @@ func (d *disk) validateConfig(instConf instance.ConfigReader) error {
// type: string
// default: `virtio-scsi`
// required: no
// shortdesc: Only for VMs: Override the bus for the device (`nvme`, `virtio-blk`, or `virtio-scsi`)
"io.bus": validate.Optional(validate.IsOneOf("nvme", "virtio-blk", "virtio-scsi")),
// shortdesc: Only for VMs: Override the bus for the device (`nvme`, `virtio-blk`, `virtio-scsi`, `auto`, `9p`, or `virtiofs`)
"io.bus": validate.Optional(validate.IsOneOf("nvme", "virtio-blk", "virtio-scsi", "auto", "9p", "virtiofs")),
}

err := d.config.Validate(rules)
Expand Down Expand Up @@ -1226,79 +1226,91 @@ func (d *disk) startVM() (*deviceConfig.RunConfig, error) {
rawIDMaps.Entries = diskAddRootUserNSEntry(rawIDMaps.Entries, 65534)
}

busOption, cacheOption := d.config["io.bus"], d.config["io.cache"]

if !(d.config["io.bus"] == "auto" || d.config["io.bus"] == "9p" || d.config["io.bus"] == "virtiofs") {
busOption = "auto"
}

// Start virtiofsd for virtio-fs share. The agent prefers to use this over the
// virtfs-proxy-helper 9p share. The 9p share will only be used as a fallback.
err = func() error {
sockPath, pidPath := d.vmVirtiofsdPaths()
logPath := filepath.Join(d.inst.LogPath(), fmt.Sprintf("disk.%s.log", d.name))
_ = os.Remove(logPath) // Remove old log if needed.
if busOption == "auto" || busOption == "virtiofs" {
err = func() error {
sockPath, pidPath := d.vmVirtiofsdPaths()
logPath := filepath.Join(d.inst.LogPath(), fmt.Sprintf("disk.%s.log", d.name))
_ = os.Remove(logPath) // Remove old log if needed.

revertFunc, unixListener, err := DiskVMVirtiofsdStart(d.state.OS.ExecPath, d.inst, sockPath, pidPath, logPath, mount.DevPath, rawIDMaps.Entries)
if err != nil {
var errUnsupported UnsupportedError
if errors.As(err, &errUnsupported) {
d.logger.Warn("Unable to use virtio-fs for device, using 9p as a fallback", logger.Ctx{"err": errUnsupported})

if errUnsupported == ErrMissingVirtiofsd {
_ = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
return tx.UpsertWarningLocalNode(ctx, d.inst.Project().Name, cluster.TypeInstance, d.inst.ID(), warningtype.MissingVirtiofsd, "Using 9p as a fallback")
})
} else {
// Resolve previous warning.
_ = warnings.ResolveWarningsByLocalNodeAndProjectAndType(d.state.DB.Cluster, d.inst.Project().Name, warningtype.MissingVirtiofsd)
revertFunc, unixListener, err := DiskVMVirtiofsdStart(d.state.OS.ExecPath, d.inst, sockPath, pidPath, logPath, mount.DevPath, rawIDMaps.Entries, cacheOption)
if err != nil {
if busOption == "auto" {
var errUnsupported UnsupportedError
if errors.As(err, &errUnsupported) {
d.logger.Warn("Unable to use virtio-fs for device, using 9p as a fallback", logger.Ctx{"err": errUnsupported})

if errUnsupported == ErrMissingVirtiofsd {
_ = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
return tx.UpsertWarningLocalNode(ctx, d.inst.Project().Name, cluster.TypeInstance, d.inst.ID(), warningtype.MissingVirtiofsd, "Using 9p as a fallback")
})
} else {
// Resolve previous warning.
_ = warnings.ResolveWarningsByLocalNodeAndProjectAndType(d.state.DB.Cluster, d.inst.Project().Name, warningtype.MissingVirtiofsd)
}

return nil
}
}

return nil
return err
}

return err
}

revert.Add(revertFunc)
revert.Add(revertFunc)

// Request the unix listener is closed after QEMU has connected on startup.
runConf.PostHooks = append(runConf.PostHooks, unixListener.Close)
// Request the unix listener is closed after QEMU has connected on startup.
runConf.PostHooks = append(runConf.PostHooks, unixListener.Close)

// Resolve previous warning
_ = warnings.ResolveWarningsByLocalNodeAndProjectAndType(d.state.DB.Cluster, d.inst.Project().Name, warningtype.MissingVirtiofsd)
// Resolve previous warning
_ = warnings.ResolveWarningsByLocalNodeAndProjectAndType(d.state.DB.Cluster, d.inst.Project().Name, warningtype.MissingVirtiofsd)

// Add the socket path to the mount options to indicate to the qemu driver
// that this share is available.
// Note: the sockPath is not passed to the QEMU via mount.DevPath like the
// 9p share above. This is because we run the 9p share concurrently
// and can only pass one DevPath at a time. Instead pass the sock path to
// the QEMU driver via the mount opts field as virtiofsdSock to allow the
// QEMU driver also setup the virtio-fs share.
mount.Opts = append(mount.Opts, fmt.Sprintf("%s=%s", DiskVirtiofsdSockMountOpt, sockPath))
// Add the socket path to the mount options to indicate to the qemu driver
// that this share is available.
// Note: the sockPath is not passed to the QEMU via mount.DevPath like the
// 9p share above. This is because we run the 9p share concurrently
// and can only pass one DevPath at a time. Instead pass the sock path to
// the QEMU driver via the mount opts field as virtiofsdSock to allow the
// QEMU driver also setup the virtio-fs share.
mount.Opts = append(mount.Opts, fmt.Sprintf("%s=%s", DiskVirtiofsdSockMountOpt, sockPath))

return nil
}()
if err != nil {
return nil, fmt.Errorf("Failed to setup virtiofsd for device %q: %w", d.name, err)
return nil
}()
if err != nil {
return nil, fmt.Errorf("Failed to setup virtiofsd for device %q: %w", d.name, err)
}
}

// We can't hotplug 9p shares, so only do 9p for stopped instances.
if !d.inst.IsRunning() {
// Start virtfs-proxy-helper for 9p share (this will rewrite mount.DevPath with
// socket FD number so must come after starting virtiofsd).
err = func() error {
sockFile, cleanup, err := DiskVMVirtfsProxyStart(d.state.OS.ExecPath, d.vmVirtfsProxyHelperPaths(), mount.DevPath, rawIDMaps.Entries)
if err != nil {
return err
}
if busOption == "auto" || busOption == "9p" {
err = func() error {
sockFile, cleanup, err := DiskVMVirtfsProxyStart(d.state.OS.ExecPath, d.vmVirtfsProxyHelperPaths(), mount.DevPath, rawIDMaps.Entries)
if err != nil {
return err
}

revert.Add(cleanup)
revert.Add(cleanup)

// Request the unix socket is closed after QEMU has connected on startup.
runConf.PostHooks = append(runConf.PostHooks, sockFile.Close)
// Request the unix socket is closed after QEMU has connected on startup.
runConf.PostHooks = append(runConf.PostHooks, sockFile.Close)

// Use 9p socket FD number as dev path so qemu can connect to the proxy.
mount.DevPath = fmt.Sprintf("%d", sockFile.Fd())
// Use 9p socket FD number as dev path so qemu can connect to the proxy.
mount.DevPath = fmt.Sprintf("%d", sockFile.Fd())

return nil
}()
if err != nil {
return nil, fmt.Errorf("Failed to setup virtfs-proxy-helper for device %q: %w", d.name, err)
return nil
}()
if err != nil {
return nil, fmt.Errorf("Failed to setup virtfs-proxy-helper for device %q: %w", d.name, err)
}
}
}
} else {
Expand Down

0 comments on commit 5cc1e00

Please sign in to comment.