Initial implementation of volume plugins

This implements support for mounting and unmounting volumes
backed by volume plugins. Support for actually retrieving
plugins requires a pull request to land in containers.conf and
then that to be vendored, and as such is not yet ready. Given
this, this code is only compile tested. However, the code for
everything past retrieving the plugin has been written - there is
support for creating, removing, mounting, and unmounting volumes,
which should allow full functionality once the c/common PR is
merged.

A major change is the signature of the MountPoint function for
volumes, which now, by necessity, returns an error. Named volumes
managed by a plugin do not have a mountpoint we control; instead,
it is managed entirely by the plugin. As such, we need to cache
the path in the DB, and calls to retrieve it now need to access
the DB (and may fail as such).

Notably absent is support for SELinux relabelling and chowning
these volumes. Given that we don't manage the mountpoint for
these volumes, I am extremely reluctant to try and modify it - we
could easily break the plugin trying to chown or relabel it.

Also, we had no less than *5* separate implementations of
inspecting a volume floating around in pkg/infra/abi and
pkg/api/handlers/libpod. And none of them used volume.Inspect(),
the only correct way of inspecting volumes. Remove them all and
consolidate to using the correct way. Compat API is likely still
doing things the wrong way, but that is an issue for another day.

Fixes containers#4304

Signed-off-by: Matthew Heon <matthew.heon@pm.me>

docs/source/markdown/podman-volume-create.1.md

@@ -17,7 +17,7 @@ driver options can be set using the **--opt** flag.
 
 #### **--driver**=*driver*
 
-Specify the volume driver name (default local).
+Specify the volume driver name (default **local**). Setting this to a value other than **local** Podman will attempt to create the volume using a volume plugin with the given name. Such plugins must be defined in the **volume_plugins** section of the **containers.conf** configuration file.
 
 #### **--help**
 
@@ -30,13 +30,14 @@ Set metadata for a volume (e.g., --label mykey=value).
 #### **--opt**=*option*, **-o**
 
 Set driver specific options.
-For the default driver, `local`, this allows a volume to be configured to mount a filesystem on the host.
+For the default driver, **local**, this allows a volume to be configured to mount a filesystem on the host.
 For the `local` driver the following options are supported: `type`, `device`, and `o`.
 The `type` option sets the type of the filesystem to be mounted, and is equivalent to the `-t` flag to **mount(8)**.
 The `device` option sets the device to be mounted, and is equivalent to the `device` argument to **mount(8)**.
 The `o` option sets options for the mount, and is equivalent to the `-o` flag to **mount(8)** with two exceptions.
 The `o` option supports `uid` and `gid` options to set the UID and GID of the created volume that are not normally supported by **mount(8)**.
-Using volume options with the `local` driver requires root privileges.
+Using volume options with the **local** driver requires root privileges.
+When not using the **local** driver, the given options will be passed directly to the volume plugin. In this case, supported options will be dictated by the plugin in question, not Podman.
 
 ## EXAMPLES
 
@@ -53,7 +54,8 @@ $ podman volume create --label foo=bar myvol
 ```
 
 ## SEE ALSO
-podman-volume(1), mount(8)
+**podman-volume**(1), **mount**(8), **containers.conf**(5)
 
 ## HISTORY
+January 2020, updated with information on volume plugins by Matthew Heon <mheon@redhat.com>
 November 2018, Originally compiled by Urvashi Mohnani <umohnani@redhat.com>

libpod/boltdb_state.go

@@ -304,6 +304,7 @@ func (s *BoltState) Refresh() error {
 
 			// Reset mount count to 0
 			oldState.MountCount = 0
+			oldState.MountPoint = ""
 
 			newState, err := json.Marshal(oldState)
 			if err != nil {

libpod/boltdb_state_internal.go

@@ -497,6 +497,16 @@ func (s *BoltState) getVolumeFromDB(name []byte, volume *Volume, volBkt *bolt.Bu
 		}
 	}
 
+	// Retrieve volume driver
+	if volume.UsesVolumeDriver() {
+		plugin, err := s.runtime.getVolumePlugin(volume.config.Driver)
+		if err != nil {
+			logrus.Errorf("Volume %s uses volume plugin %s, but it cannot be accessed - some functionality may not be available: %v", volume.Name(), volume.config.Driver, err)
+		} else {
+			volume.plugin = plugin
+		}
+	}
+
 	// Get the lock
 	lock, err := s.runtime.lockManager.RetrieveLock(volume.config.LockID)
 	if err != nil {

libpod/container_inspect.go

@@ -212,7 +212,12 @@ func (c *Container) getInspectMounts(namedVolumes []*ContainerNamedVolume, image
 			return nil, errors.Wrapf(err, "error looking up volume %s in container %s config", volume.Name, c.ID())
 		}
 		mountStruct.Driver = volFromDB.Driver()
-		mountStruct.Source = volFromDB.MountPoint()
+
+		mountPoint, err := volFromDB.MountPoint()
+		if err != nil {
+			return nil, err
+		}
+		mountStruct.Source = mountPoint
 
 		parseMountOptionsForInspect(volume.Options, &mountStruct)
 

libpod/container_internal.go

@@ -1535,8 +1535,18 @@ func (c *Container) mountNamedVolume(v *ContainerNamedVolume, mountpoint string)
 			return nil, err
 		}
 
+		// HACK HACK HACK - copy up into a volume driver is 100% broken
+		// right now.
+		if vol.UsesVolumeDriver() {
+			logrus.Infof("Not copying up into volume %s as it uses a volume driver", vol.Name())
+			return vol, nil
+		}
+
 		// If the volume is not empty, we should not copy up.
-		volMount := vol.MountPoint()
+		volMount, err := vol.MountPoint()
+		if err != nil {
+			return nil, err
+		}
 		contents, err := ioutil.ReadDir(volMount)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error listing contents of volume %s mountpoint when copying up from container %s", vol.Name(), c.ID())
@@ -1574,7 +1584,11 @@ func (c *Container) chownVolume(volumeName string) error {
 		return err
 	}
 
-	if vol.state.NeedsChown {
+	// TODO: For now, I've disabled chowning volumes owned by non-Podman
+	// drivers. This may be safe, but it's really going to be a case-by-case
+	// thing, I think - safest to leave disabled now and reenable later if
+	// there is a demand.
+	if vol.state.NeedsChown && !vol.UsesVolumeDriver() {
 		vol.state.NeedsChown = false
 
 		uid := int(c.config.Spec.Process.User.UID)
@@ -1601,7 +1615,10 @@ func (c *Container) chownVolume(volumeName string) error {
 			return err
 		}
 
-		mountPoint := vol.MountPoint()
+		mountPoint, err := vol.MountPoint()
+		if err != nil {
+			return err
+		}
 
 		if err := os.Lchown(mountPoint, uid, gid); err != nil {
 			return err

libpod/container_internal_linux.go

@@ -342,7 +342,10 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "error retrieving volume %s to add to container %s", namedVol.Name, c.ID())
 		}
-		mountPoint := volume.MountPoint()
+		mountPoint, err := volume.MountPoint()
+		if err != nil {
+			return nil, err
+		}
 		volMount := spec.Mount{
 			Type:        "bind",
 			Source:      mountPoint,

libpod/define/errors.go

@@ -34,6 +34,10 @@ var (
 	// aliases.
 	ErrNoAliases = errors.New("no aliases for container")
 
+	// ErrMissingPlugin indicates that the requested operation requires a
+	// plugin that is not present on the system or in the configuration.
+	ErrMissingPlugin = errors.New("required plugin missing")
+
 	// ErrCtrExists indicates a container with the same name or ID already
 	// exists
 	ErrCtrExists = errors.New("container already exists")

libpod/define/volume_inspect.go

@@ -0,0 +1,47 @@
+package define
+
+import (
+	"time"
+)
+
+// InspectVolumeData is the output of Inspect() on a volume. It is matched to
+// the format of 'docker volume inspect'.
+type InspectVolumeData struct {
+	// Name is the name of the volume.
+	Name string `json:"Name"`
+	// Driver is the driver used to create the volume.
+	// If set to "local" or "", the Local driver (Podman built-in code) is
+	// used to service the volume; otherwise, a volume plugin with the given
+	// name is used to mount and manage the volume.
+	Driver string `json:"Driver"`
+	// Mountpoint is the path on the host where the volume is mounted.
+	Mountpoint string `json:"Mountpoint"`
+	// CreatedAt is the date and time the volume was created at. This is not
+	// stored for older Libpod volumes; if so, it will be omitted.
+	CreatedAt time.Time `json:"CreatedAt,omitempty"`
+	// Status is used to return information on the volume's current state,
+	// if the volume was created using a volume plugin (uses a Driver that
+	// is not the local driver).
+	// Status is provided to us by an external program, so no guarantees are
+	// made about its format or contents. Further, it is an optional field,
+	// so it may not be set even in cases where a volume plugin is in use.
+	Status map[string]interface{} `json:"Status,omitempty"`
+	// Labels includes the volume's configured labels, key:value pairs that
+	// can be passed during volume creation to provide information for third
+	// party tools.
+	Labels map[string]string `json:"Labels"`
+	// Scope is unused and provided solely for Docker compatibility. It is
+	// unconditionally set to "local".
+	Scope string `json:"Scope"`
+	// Options is a set of options that were used when creating the volume.
+	// It is presently not used.
+	Options map[string]string `json:"Options"`
+	// UID is the UID that the volume was created with.
+	UID int `json:"UID,omitempty"`
+	// GID is the GID that the volume was created with.
+	GID int `json:"GID,omitempty"`
+	// Anonymous indicates that the volume was created as an anonymous
+	// volume for a specific container, and will be be removed when any
+	// container using it is removed.
+	Anonymous bool `json:"Anonymous,omitempty"`
+}

libpod/options.go

@@ -1549,17 +1549,6 @@ func WithVolumeDriver(driver string) VolumeCreateOption {
 			return define.ErrVolumeFinalized
 		}
 
-		// Uncomment when volume plugins are ready for use.
-		// if driver != define.VolumeDriverLocal {
-		// 	if _, err := plugin.GetVolumePlugin(driver); err != nil {
-		// 		return err
-		// 	}
-		// }
-
-		if driver != define.VolumeDriverLocal {
-			return define.ErrNotImplemented
-		}
-
 		volume.config.Driver = driver
 		return nil
 	}

libpod/plugin/volume_api.go

@@ -2,8 +2,9 @@ package plugin
 
 import (
 	"bytes"
-	"fmt"
+	"context"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -43,7 +44,6 @@ var (
 
 const (
 	defaultTimeout   = 5 * time.Second
-	defaultPath      = "/run/docker/plugins"
 	volumePluginType = "VolumeDriver"
 )
 
@@ -64,6 +64,8 @@ type VolumePlugin struct {
 	Name string
 	// SocketPath is the unix socket at which the plugin is accessed.
 	SocketPath string
+	// Client is the HTTP client we use to connect to the plugin.
+	Client *http.Client
 }
 
 // This is the response from the activate endpoint of the API.
@@ -76,17 +78,15 @@ type activateResponse struct {
 func validatePlugin(newPlugin *VolumePlugin) error {
 	// It's a socket. Is it a plugin?
 	// Hit the Activate endpoint to find out if it is, and if so what kind
-	req, err := http.NewRequest("POST", activatePath, nil)
+	req, err := http.NewRequest("POST", "http://plugin"+activatePath, nil)
 	if err != nil {
 		return errors.Wrapf(err, "error making request to volume plugin %s activation endpoint", newPlugin.Name)
 	}
 
 	req.Header.Set("Host", newPlugin.getURI())
 	req.Header.Set("Content-Type", sdk.DefaultContentTypeV1_1)
 
-	client := new(http.Client)
-	client.Timeout = defaultTimeout
-	resp, err := client.Do(req)
+	resp, err := newPlugin.Client.Do(req)
 	if err != nil {
 		return errors.Wrapf(err, "error sending request to plugin %s activation endpoint", newPlugin.Name)
 	}
@@ -121,30 +121,49 @@ func validatePlugin(newPlugin *VolumePlugin) error {
 		return errors.Wrapf(ErrNotVolumePlugin, "plugin %s does not implement volume plugin, instead provides %s", newPlugin.Name, strings.Join(respStruct.Implements, ", "))
 	}
 
+	if plugins == nil {
+		plugins = make(map[string]*VolumePlugin)
+	}
+
 	plugins[newPlugin.Name] = newPlugin
 
 	return nil
 }
 
-// GetVolumePlugin gets a single volume plugin by path.
-// TODO: We should not be auto-completing based on a default path; we should
-// require volumes to have been pre-specified in containers.conf (will need a
-// function to pre-populate the plugins list, and we should probably do a lazy
-// initialization there to not slow things down too much).
-func GetVolumePlugin(name string) (*VolumePlugin, error) {
+// GetVolumePlugin gets a single volume plugin, with the given name, at the
+// given path.
+func GetVolumePlugin(name string, path string) (*VolumePlugin, error) {
 	pluginsLock.Lock()
 	defer pluginsLock.Unlock()
 
 	plugin, exists := plugins[name]
 	if exists {
+		// This shouldn't be possible, but just in case...
+		if plugin.SocketPath != filepath.Clean(path) {
+			return nil, errors.Wrapf(define.ErrInvalidArg, "requested path %q for volume plugin %s does not match pre-existing path for plugin, %q", path, name, plugin.SocketPath)
+		}
+
 		return plugin, nil
 	}
 
 	// It's not cached. We need to get it.
 
 	newPlugin := new(VolumePlugin)
 	newPlugin.Name = name
-	newPlugin.SocketPath = filepath.Join(defaultPath, fmt.Sprintf("%s.sock", name))
+	newPlugin.SocketPath = filepath.Clean(path)
+
+	// Need an HTTP client to force a Unix connection.
+	// And since we can reuse it, might as well cache it.
+	client := new(http.Client)
+	client.Timeout = defaultTimeout
+	// This bit borrowed from pkg/bindings/connection.go
+	client.Transport = &http.Transport{
+		DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+			return (&net.Dialer{}).DialContext(ctx, "unix", newPlugin.SocketPath)
+		},
+		DisableCompression: true,
+	}
+	newPlugin.Client = client
 
 	stat, err := os.Stat(newPlugin.SocketPath)
 	if err != nil {
@@ -183,6 +202,7 @@ func (p *VolumePlugin) verifyReachable() error {
 }
 
 // Send a request to the volume plugin for handling.
+// Callers *MUST* close the response when they are done.
 func (p *VolumePlugin) sendRequest(toJSON interface{}, hasBody bool, endpoint string) (*http.Response, error) {
 	var (
 		reqJSON []byte
@@ -196,21 +216,20 @@ func (p *VolumePlugin) sendRequest(toJSON interface{}, hasBody bool, endpoint st
 		}
 	}
 
-	req, err := http.NewRequest("POST", endpoint, bytes.NewReader(reqJSON))
+	req, err := http.NewRequest("POST", "http://plugin"+endpoint, bytes.NewReader(reqJSON))
 	if err != nil {
 		return nil, errors.Wrapf(err, "error making request to volume plugin %s endpoint %s", p.Name, endpoint)
 	}
 
 	req.Header.Set("Host", p.getURI())
 	req.Header.Set("Content-Type", sdk.DefaultContentTypeV1_1)
 
-	client := new(http.Client)
-	client.Timeout = defaultTimeout
-	resp, err := client.Do(req)
+	resp, err := p.Client.Do(req)
 	if err != nil {
 		return nil, errors.Wrapf(err, "error sending request to volume plugin %s endpoint %s", p.Name, endpoint)
 	}
-	defer resp.Body.Close()
+	// We are *deliberately not closing* response here. It is the
+	// responsibility of the caller to do so after reading the response.
 
 	return resp, nil
 }

libpod/runtime.go

@@ -17,6 +17,7 @@ import (
 	"github.com/containers/podman/v2/libpod/events"
 	"github.com/containers/podman/v2/libpod/image"
 	"github.com/containers/podman/v2/libpod/lock"
+	"github.com/containers/podman/v2/libpod/plugin"
 	"github.com/containers/podman/v2/libpod/shutdown"
 	"github.com/containers/podman/v2/pkg/cgroups"
 	"github.com/containers/podman/v2/pkg/registries"
@@ -888,3 +889,19 @@ func (r *Runtime) reloadStorageConf() error {
 	logrus.Infof("applied new storage configuration: %v", r.storageConfig)
 	return nil
 }
+
+// getVolumePlugin gets a specific volume plugin given its name.
+// TODO: This needs containers.conf work to merge and be vendored first.
+func (r *Runtime) getVolumePlugin(name string) (*plugin.VolumePlugin, error) {
+	// There is no plugin for local.
+	if name == define.VolumeDriverLocal || name == "" {
+		return nil, nil
+	}
+
+	pluginPath, ok := r.config.Engine.VolumePlugins[name]
+	if !ok {
+		return nil, errors.Wrapf(define.ErrMissingPlugin, "no volume plugin with name %s available", name)
+	}
+
+	return plugin.GetVolumePlugin(name, pluginPath)
+}