diff --git a/go.mod b/go.mod index dd95c586f82b..55092e27288b 100644 --- a/go.mod +++ b/go.mod @@ -144,3 +144,5 @@ require ( ) replace github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.1-0.20220617142545-8b9452f75cbc + +replace github.com/containers/storage => github.com/mtrmac/storage v0.0.0-20221201150117-26d820ccdc9e diff --git a/go.sum b/go.sum index c9eebef20e45..59683a25910a 100644 --- a/go.sum +++ b/go.sum @@ -54,8 +54,6 @@ github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935 github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -67,7 +65,6 @@ github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugX github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= @@ -80,9 +77,7 @@ github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2 github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.8.22/go.mod h1:91uVCVzvX2QD16sMCenoxxXo6L1wJnLMX2PSufFMtF0= github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= -github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim v0.9.5 h1:AbV+VPfTrIVffukazHcpxmz/sRiE6YaMDzHWR9BXZHo= github.com/Microsoft/hcsshim v0.9.5/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= @@ -234,8 +229,6 @@ github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFY github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= -github.com/containerd/stargz-snapshotter/estargz v0.9.0/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0= -github.com/containerd/stargz-snapshotter/estargz v0.12.0/go.mod h1:AIQ59TewBFJ4GOPEQXujcrJ/EKxh5xXZegW1rkR1P/M= github.com/containerd/stargz-snapshotter/estargz v0.13.0 h1:fD7AwuVV+B40p0d9qVkH/Au1qhp8hn/HWJHIYjpEcfw= github.com/containerd/stargz-snapshotter/estargz v0.13.0/go.mod h1:m+9VaGJGlhCnrcEUod8mYumTmRgblwd3rC5UCEh2Yp0= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= @@ -279,10 +272,6 @@ github.com/containers/ocicrypt v1.1.6 h1:uoG52u2e91RE4UqmBICZY8dNshgfvkdl3BW6jnx github.com/containers/ocicrypt v1.1.6/go.mod h1:WgjxPWdTJMqYMjf3M6cuIFFA1/MpyyhIM99YInA+Rvc= github.com/containers/psgo v1.8.0 h1:2loGekmGAxM9ir5OsXWEfGwFxorMPYnc6gEDsGFQvhY= github.com/containers/psgo v1.8.0/go.mod h1:T8ZxnX3Ur4RvnhxFJ7t8xJ1F48RhiZB4rSrOaR/qGHc= -github.com/containers/storage v1.37.0/go.mod h1:kqeJeS0b7DO2ZT1nVWs0XufrmPFbgV3c+Q/45RlH6r4= -github.com/containers/storage v1.43.0/go.mod h1:uZ147thiIFGdVTjMmIw19knttQnUCl3y9zjreHrg11s= -github.com/containers/storage v1.44.1-0.20221201083122-c5a80ad65f42 h1:lba+h0VcMGvO/C4Q+oMhGxpgajzyQifbcedOYQNVRx8= -github.com/containers/storage v1.44.1-0.20221201083122-c5a80ad65f42/go.mod h1:pYkSXaKIGAuEQmIf/melI5wbS/JBM++6Xp4JuVTqY7U= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= @@ -611,9 +600,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -702,6 +688,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/mtrmac/storage v0.0.0-20221201150117-26d820ccdc9e h1:EOHXuecg4HkAYmh1ieSeNmNhkGXQQwBfQKFk/RIf21M= +github.com/mtrmac/storage v0.0.0-20221201150117-26d820ccdc9e/go.mod h1:pYkSXaKIGAuEQmIf/melI5wbS/JBM++6Xp4JuVTqY7U= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -766,7 +754,6 @@ github.com/opencontainers/runtime-tools v0.9.1-0.20221014010322-58c91d646d86/go. github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= -github.com/opencontainers/selinux v1.8.5/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.10.2 h1:NFy2xCsjn7+WspbfZkUd5zyVeisV7VFbPSP96+8/ha4= @@ -1221,7 +1208,6 @@ golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index e7fe52fd246d..59ed0e46cab5 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -299,6 +299,9 @@ type rwLayerStore interface { // Clean up unreferenced layers GarbageCollect() error + + // supportsShifting() returns true if the driver.Driver.SupportsShifting(). + supportsShifting() bool } type layerStore struct { @@ -2234,6 +2237,10 @@ func (r *layerStore) LayersByUncompressedDigest(d digest.Digest) ([]Layer, error return r.layersByDigestMap(r.byuncompressedsum, d) } +func (r *layerStore) supportsShifting() bool { + return r.driver.SupportsShifting() +} + func closeAll(closes ...func() error) (rErr error) { for _, f := range closes { if err := f(); err != nil { diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index f5c2e73ac856..17df74ecb79c 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -580,31 +580,47 @@ type ContainerOptions struct { } type store struct { - lastLoaded time.Time - runRoot string - graphLock *lockfile.LockFile - usernsLock *lockfile.LockFile - graphRoot string - graphDriverName string - graphOptions []string - pullOptions map[string]string - uidMap []idtools.IDMap - gidMap []idtools.IDMap - autoUsernsUser string - additionalUIDs *idSet // Set by getAvailableIDs() - additionalGIDs *idSet // Set by getAvailableIDs() - autoNsMinSize uint32 - autoNsMaxSize uint32 + // The following fields are only set when constructing store, and must never be modified afterwards. + // They are safe to access without any other locking. + runRoot string + graphDriverName string // Initially set to the user-requested value, possibly ""; updated during store construction, and does not change afterwards. + // graphLock: + // - Ensures that we always reload graphDriver, and the primary layer store, after any process does store.Shutdown. This is necessary + // because (??) the Shutdown may forcibly unmount and clean up, affecting graph driver state in a way only a graph driver + // and layer store reinitialization can notice. + // - Ensures that store.Shutdown is exclusive with mount operations. This is necessary at because some + // graph drivers call mount.MakePrivate() during initialization, the mount operations require that, and the driver’s Cleanup() method + // may undo that. So, holding graphLock is required throughout the duration of Shutdown(), and the duration of any mount + // (but not unmount) calls. + // - Within this store object, protects access to some related in-memory state. + graphLock *lockfile.LockFile + usernsLock *lockfile.LockFile + graphRoot string + graphOptions []string + pullOptions map[string]string + uidMap []idtools.IDMap + gidMap []idtools.IDMap + autoUsernsUser string + autoNsMinSize uint32 + autoNsMaxSize uint32 + imageStore rwImageStore + roImageStores []roImageStore + containerStore rwContainerStore + digestLockRoot string + disableVolatile bool + transientStore bool + + // The following fields can only be accessed with graphLock held. graphLockLastWrite lockfile.LastWrite - graphDriver drivers.Driver - layerStore rwLayerStore - roLayerStores []roLayerStore - imageStore rwImageStore - roImageStores []roImageStore - containerStore rwContainerStore - digestLockRoot string - disableVolatile bool - transientStore bool + // FIXME: This field is only set when holding graphLock, but locking rules of the driver + // interface itself are not documented here. It is extensively used without holding graphLock. + graphDriver drivers.Driver + layerStoreUseGetters rwLayerStore // Almost all users should use the provided accessors instead of accessing this field directly. + roLayerStoresUseGetters []roLayerStore // Almost all users should use the provided accessors instead of accessing this field directly. + + // FIXME: The following fields need locking, and don’t have it. + additionalUIDs *idSet // Set by getAvailableIDs() + additionalGIDs *idSet // Set by getAvailableIDs() } // GetStore attempts to find an already-created Store object matching the @@ -699,33 +715,22 @@ func GetStore(options types.StoreOptions) (Store, error) { } s := &store{ runRoot: options.RunRoot, + graphDriverName: options.GraphDriverName, graphLock: graphLock, + usernsLock: usernsLock, graphRoot: options.GraphRoot, - graphDriverName: options.GraphDriverName, graphOptions: options.GraphDriverOptions, + pullOptions: options.PullOptions, uidMap: copyIDMap(options.UIDMap), gidMap: copyIDMap(options.GIDMap), autoUsernsUser: options.RootAutoNsUser, autoNsMinSize: autoNsMinSize, autoNsMaxSize: autoNsMaxSize, - additionalUIDs: nil, - additionalGIDs: nil, - usernsLock: usernsLock, disableVolatile: options.DisableVolatile, transientStore: options.TransientStore, - pullOptions: options.PullOptions, - } - if err := func() error { // A scope for defer - s.graphLock.Lock() - defer s.graphLock.Unlock() - lastWrite, err := s.graphLock.GetLastWrite() - if err != nil { - return err - } - s.graphLockLastWrite = lastWrite - return nil - }(); err != nil { - return nil, err + + additionalUIDs: nil, + additionalGIDs: nil, } if err := s.load(); err != nil { return nil, err @@ -796,13 +801,27 @@ func (s *store) GIDMap() []idtools.IDMap { return copyIDMap(s.gidMap) } +// This must only be called when constructing store; it writes to fields that are assumed to be constant after constrution. func (s *store) load() error { - driver, err := s.GraphDriver() - if err != nil { + var driver drivers.Driver + if err := func() error { // A scope for defer + s.graphLock.Lock() + defer s.graphLock.Unlock() + lastWrite, err := s.graphLock.GetLastWrite() + if err != nil { + return err + } + s.graphLockLastWrite = lastWrite + driver, err = s.createGraphDriverLocked() + if err != nil { + return err + } + s.graphDriver = driver + s.graphDriverName = driver.String() + return nil + }(); err != nil { return err } - s.graphDriver = driver - s.graphDriverName = driver.String() driverPrefix := s.graphDriverName + "-" gipath := filepath.Join(s.graphRoot, driverPrefix+"images") @@ -814,9 +833,6 @@ func (s *store) load() error { return err } s.imageStore = ris - if _, err := s.getROImageStores(); err != nil { - return err - } gcpath := filepath.Join(s.graphRoot, driverPrefix+"containers") if err := os.MkdirAll(gcpath, 0700); err != nil { @@ -856,10 +872,58 @@ func (s *store) GetDigestLock(d digest.Digest) (Locker, error) { return lockfile.GetLockFile(filepath.Join(s.digestLockRoot, d.String())) } -func (s *store) getGraphDriver() (drivers.Driver, error) { - if s.graphDriver != nil { - return s.graphDriver, nil +// startUsingGraphDriver obtains s.graphLock and ensures that s.graphDriver is set and fresh. +// It only intended to be used on a fully-constructed store. +// If this succeeds, the caller MUST call stopUsingGraphDriver(). +func (s *store) startUsingGraphDriver() error { + s.graphLock.Lock() + succeeded := false + defer func() { + if !succeeded { + s.graphLock.Unlock() + } + }() + + lastWrite, modified, err := s.graphLock.ModifiedSince(s.graphLockLastWrite) + if err != nil { + return err + } + if modified { + driver, err := s.createGraphDriverLocked() + if err != nil { + return err + } + // Our concurrency design requires s.graphDriverName not to be modified after + // store is constructed. + // It’s fine for driver.String() not to match the requested graph driver name + // (e.g. if the user asks for overlay2 and gets overlay), but it must be an idempotent + // mapping: + // driver1 := drivers.New(userInput, config) + // name1 := driver1.String() + // name2 := drivers.New(name1, config).String() + // assert(name1 == name2) + if s.graphDriverName != driver.String() { + return fmt.Errorf("graph driver name changed from %q to %q during reload", + s.graphDriverName, driver.String()) + } + s.graphDriver = driver + s.layerStoreUseGetters = nil + s.graphLockLastWrite = lastWrite } + + succeeded = true + return nil +} + +// stopUsingGraphDriver releases graphLock obtained by startUsingGraphDriver. +func (s *store) stopUsingGraphDriver() { + s.graphLock.Unlock() +} + +// createGraphDriverLocked creates a new instance of graph driver for s, and returns it. +// Almost all users should use startUsingGraphDriver instead. +// The caller must hold s.graphLock. +func (s *store) createGraphDriverLocked() (drivers.Driver, error) { config := drivers.Options{ Root: s.graphRoot, RunRoot: s.runRoot, @@ -867,42 +931,23 @@ func (s *store) getGraphDriver() (drivers.Driver, error) { UIDMaps: s.uidMap, GIDMaps: s.gidMap, } - driver, err := drivers.New(s.graphDriverName, config) - if err != nil { - return nil, err - } - s.graphDriver = driver - s.graphDriverName = driver.String() - return driver, nil + return drivers.New(s.graphDriverName, config) } func (s *store) GraphDriver() (drivers.Driver, error) { - s.graphLock.Lock() - defer s.graphLock.Unlock() - if s.graphLock.TouchedSince(s.lastLoaded) { - s.graphDriver = nil - s.layerStore = nil - s.lastLoaded = time.Now() + if err := s.startUsingGraphDriver(); err != nil { + return nil, err } - return s.getGraphDriver() + defer s.stopUsingGraphDriver() + return s.graphDriver, nil } -// getLayerStore obtains and returns a handle to the writeable layer store object +// getLayerStoreLocked obtains and returns a handle to the writeable layer store object // used by the Store. -func (s *store) getLayerStore() (rwLayerStore, error) { - s.graphLock.Lock() - defer s.graphLock.Unlock() - if s.graphLock.TouchedSince(s.lastLoaded) { - s.graphDriver = nil - s.layerStore = nil - s.lastLoaded = time.Now() - } - if s.layerStore != nil { - return s.layerStore, nil - } - driver, err := s.getGraphDriver() - if err != nil { - return nil, err +// It must be called with s.graphLock held. +func (s *store) getLayerStoreLocked() (rwLayerStore, error) { + if s.layerStoreUseGetters != nil { + return s.layerStoreUseGetters, nil } driverPrefix := s.graphDriverName + "-" rlpath := filepath.Join(s.runRoot, driverPrefix+"layers") @@ -913,56 +958,94 @@ func (s *store) getLayerStore() (rwLayerStore, error) { if err := os.MkdirAll(glpath, 0700); err != nil { return nil, err } - rls, err := s.newLayerStore(rlpath, glpath, driver, s.transientStore) + rls, err := s.newLayerStore(rlpath, glpath, s.graphDriver, s.transientStore) if err != nil { return nil, err } - s.layerStore = rls - return s.layerStore, nil + s.layerStoreUseGetters = rls + return s.layerStoreUseGetters, nil +} + +// getLayerStore obtains and returns a handle to the writeable layer store object +// used by the store. +// It must be called WITHOUT s.graphLock held. +func (s *store) getLayerStore() (rwLayerStore, error) { + if err := s.startUsingGraphDriver(); err != nil { + return nil, err + } + defer s.stopUsingGraphDriver() + return s.getLayerStoreLocked() } // getROLayerStores obtains additional read/only layer store objects used by the // Store. -func (s *store) getROLayerStores() ([]roLayerStore, error) { - s.graphLock.Lock() - defer s.graphLock.Unlock() - if s.roLayerStores != nil { - return s.roLayerStores, nil - } - driver, err := s.getGraphDriver() - if err != nil { - return nil, err +// It must be called with s.graphLock held. +func (s *store) getROLayerStoresLocked() ([]roLayerStore, error) { + if s.roLayerStoresUseGetters != nil { + return s.roLayerStoresUseGetters, nil } driverPrefix := s.graphDriverName + "-" rlpath := filepath.Join(s.runRoot, driverPrefix+"layers") if err := os.MkdirAll(rlpath, 0700); err != nil { return nil, err } - for _, store := range driver.AdditionalImageStores() { + for _, store := range s.graphDriver.AdditionalImageStores() { glpath := filepath.Join(store, driverPrefix+"layers") - rls, err := newROLayerStore(rlpath, glpath, driver) + rls, err := newROLayerStore(rlpath, glpath, s.graphDriver) if err != nil { return nil, err } - s.roLayerStores = append(s.roLayerStores, rls) + s.roLayerStoresUseGetters = append(s.roLayerStoresUseGetters, rls) } - return s.roLayerStores, nil + return s.roLayerStoresUseGetters, nil } -// allLayerStores returns a list of all layer store objects used by the Store. -// This is a convenience method for read-only users of the Store. -func (s *store) allLayerStores() ([]roLayerStore, error) { - primary, err := s.getLayerStore() +// bothLayerStoreKindsLocked returns the primary, and additional read-only, layer store objects used by the store. +// It must be called with s.graphLock held. +func (s *store) bothLayerStoreKindsLocked() (rwLayerStore, []roLayerStore, error) { + primary, err := s.getLayerStoreLocked() if err != nil { - return nil, fmt.Errorf("loading primary layer store data: %w", err) + return nil, nil, fmt.Errorf("loading primary layer store data: %w", err) } - additional, err := s.getROLayerStores() + additional, err := s.getROLayerStoresLocked() if err != nil { - return nil, fmt.Errorf("loading additional layer stores: %w", err) + return nil, nil, fmt.Errorf("loading additional layer stores: %w", err) + } + return primary, additional, nil +} + +// bothLayerStoreKinds returns the primary, and additional read-only, layer store objects used by the store. +// It must be called with s.graphLock held. +func (s *store) bothLayerStoreKinds() (rwLayerStore, []roLayerStore, error) { + if err := s.startUsingGraphDriver(); err != nil { + return nil, nil, err + } + defer s.stopUsingGraphDriver() + return s.bothLayerStoreKindsLocked() +} + +// allLayerStores returns a list of all layer store objects used by the Store. +// This is a convenience method for read-only users of the Store. +// It must be called with s.graphLock held. +func (s *store) allLayerStoresLocked() ([]roLayerStore, error) { + primary, additional, err := s.bothLayerStoreKindsLocked() + if err != nil { + return nil, err } return append([]roLayerStore{primary}, additional...), nil } +// allLayerStores returns a list of all layer store objects used by the Store. +// This is a convenience method for read-only users of the Store. +// It must be called WITHOUT s.graphLock held. +func (s *store) allLayerStores() ([]roLayerStore, error) { + if err := s.startUsingGraphDriver(); err != nil { + return nil, err + } + defer s.stopUsingGraphDriver() + return s.allLayerStoresLocked() +} + // readAllLayerStores processes allLayerStores() in order: // It locks the store for reading, checks for updates, and calls // @@ -1016,37 +1099,10 @@ func (s *store) writeToLayerStore(fn func(store rwLayerStore) error) error { return fn(store) } -// getImageStore obtains and returns a handle to the writable image store object -// used by the Store. -func (s *store) getImageStore() (rwImageStore, error) { - if s.imageStore != nil { - return s.imageStore, nil - } - return nil, ErrLoadError -} - -// getROImageStores obtains additional read/only image store objects used by the -// Store. -func (s *store) getROImageStores() ([]roImageStore, error) { - if s.imageStore == nil { - return nil, ErrLoadError - } - - return s.roImageStores, nil -} - // allImageStores returns a list of all image store objects used by the Store. // This is a convenience method for read-only users of the Store. -func (s *store) allImageStores() ([]roImageStore, error) { - primary, err := s.getImageStore() - if err != nil { - return nil, fmt.Errorf("loading primary image store data: %w", err) - } - additional, err := s.getROImageStores() - if err != nil { - return nil, fmt.Errorf("loading additional image stores: %w", err) - } - return append([]roImageStore{primary}, additional...), nil +func (s *store) allImageStores() []roImageStore { + return append([]roImageStore{s.imageStore}, s.roImageStores...) } // readAllImageStores processes allImageStores() in order: @@ -1069,11 +1125,7 @@ func (s *store) allImageStores() ([]roImageStore, error) { // return res, err // } func (s *store) readAllImageStores(fn func(store roImageStore) (bool, error)) (bool, error) { - ImageStores, err := s.allImageStores() - if err != nil { - return true, err - } - for _, s := range ImageStores { + for _, s := range s.allImageStores() { store := s if err := store.startReading(); err != nil { return true, err @@ -1087,81 +1139,57 @@ func (s *store) readAllImageStores(fn func(store roImageStore) (bool, error)) (b } // writeToImageStore is a convenience helper for working with store.getImageStore(): -// It locks the store for writing, checks for updates, and calls fn() +// It locks the store for writing, checks for updates, and calls fn(), which can then access store.imageStore. // It returns the return value of fn, or its own error initializing the store. -func (s *store) writeToImageStore(fn func(store rwImageStore) error) error { - store, err := s.getImageStore() - if err != nil { - return err - } - - if err := store.startWriting(); err != nil { +func (s *store) writeToImageStore(fn func() error) error { + if err := s.imageStore.startWriting(); err != nil { return err } - defer store.stopWriting() - return fn(store) -} - -// getContainerStore obtains and returns a handle to the container store object -// used by the Store. -func (s *store) getContainerStore() (rwContainerStore, error) { - if s.containerStore != nil { - return s.containerStore, nil - } - return nil, ErrLoadError + defer s.imageStore.stopWriting() + return fn() } // writeToContainerStore is a convenience helper for working with store.getContainerStore(): -// It locks the store for writing, checks for updates, and calls fn() +// It locks the store for writing, checks for updates, and calls fn(), which can then access store.containerStore. // It returns the return value of fn, or its own error initializing the store. -func (s *store) writeToContainerStore(fn func(store rwContainerStore) error) error { - store, err := s.getContainerStore() - if err != nil { - return err - } - - if err := store.startWriting(); err != nil { +func (s *store) writeToContainerStore(fn func() error) error { + if err := s.containerStore.startWriting(); err != nil { return err } - defer store.stopWriting() - return fn(store) + defer s.containerStore.stopWriting() + return fn() } // writeToAllStores is a convenience helper for writing to all three stores: -// It locks the stores for writing, checks for updates, and calls fn(). +// It locks the stores for writing, checks for updates, and calls fn(), which can then access the provided layer store, +// s.imageStore and s.containerStore. // It returns the return value of fn, or its own error initializing the stores. -func (s *store) writeToAllStores(fn func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error) error { +func (s *store) writeToAllStores(fn func(rlstore rwLayerStore) error) error { rlstore, err := s.getLayerStore() if err != nil { return err } - ristore, err := s.getImageStore() - if err != nil { - return err - } - rcstore, err := s.getContainerStore() - if err != nil { - return err - } if err := rlstore.startWriting(); err != nil { return err } defer rlstore.stopWriting() - if err := ristore.startWriting(); err != nil { + if err := s.imageStore.startWriting(); err != nil { return err } - defer ristore.stopWriting() - if err := rcstore.startWriting(); err != nil { + defer s.imageStore.stopWriting() + if err := s.containerStore.startWriting(); err != nil { return err } - defer rcstore.stopWriting() + defer s.containerStore.stopWriting() - return fn(rlstore, ristore, rcstore) + return fn(rlstore) } -func (s *store) canUseShifting(uidmap, gidmap []idtools.IDMap) bool { - if s.graphDriver == nil || !s.graphDriver.SupportsShifting() { +// canUseShifting returns ??? +// store must be locked for writing. +func canUseShifting(store rwLayerStore, uidmap, gidmap []idtools.IDMap) bool { + if !store.supportsShifting() { return false } if uidmap != nil && !idtools.IsContiguous(uidmap) { @@ -1175,15 +1203,7 @@ func (s *store) canUseShifting(uidmap, gidmap []idtools.IDMap) bool { func (s *store) PutLayer(id, parent string, names []string, mountLabel string, writeable bool, options *LayerOptions, diff io.Reader) (*Layer, int64, error) { var parentLayer *Layer - rlstore, err := s.getLayerStore() - if err != nil { - return nil, -1, err - } - rlstores, err := s.getROLayerStores() - if err != nil { - return nil, -1, err - } - rcstore, err := s.getContainerStore() + rlstore, rlstores, err := s.bothLayerStoreKinds() if err != nil { return nil, -1, err } @@ -1191,10 +1211,10 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w return nil, -1, err } defer rlstore.stopWriting() - if err := rcstore.startWriting(); err != nil { + if err := s.containerStore.startWriting(); err != nil { return nil, -1, err } - defer rcstore.stopWriting() + defer s.containerStore.stopWriting() if options == nil { options = &LayerOptions{} } @@ -1226,7 +1246,7 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w return nil, -1, ErrLayerUnknown } parentLayer = ilayer - containers, err := rcstore.Containers() + containers, err := s.containerStore.Containers() if err != nil { return nil, -1, err } @@ -1253,7 +1273,7 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w OriginalDigest: options.OriginalDigest, UncompressedDigest: options.UncompressedDigest, } - if s.canUseShifting(uidMap, gidMap) { + if canUseShifting(rlstore, uidMap, gidMap) { layerOptions.IDMappingOptions = types.IDMappingOptions{HostUIDMapping: true, HostGIDMapping: true, UIDMap: nil, GIDMap: nil} } else { layerOptions.IDMappingOptions = types.IDMappingOptions{ @@ -1296,14 +1316,14 @@ func (s *store) CreateImage(id string, names []string, layer, metadata string, o } var res *Image - err := s.writeToImageStore(func(ristore rwImageStore) error { + err := s.writeToImageStore(func() error { creationDate := time.Now().UTC() if options != nil && !options.CreationDate.IsZero() { creationDate = options.CreationDate } var err error - res, err = ristore.Create(id, names, layer, metadata, creationDate, options.Digest) + res, err = s.imageStore.Create(id, names, layer, metadata, creationDate, options.Digest) return err }) return res, err @@ -1312,13 +1332,13 @@ func (s *store) CreateImage(id string, names []string, layer, metadata string, o // imageTopLayerForMapping does ??? // On entry: // - ristore must be locked EITHER for reading or writing -// - primaryImageStore must be locked for writing; it might be identical to ristore. +// - s.imageStore must be locked for writing; it might be identical to ristore. // - rlstore must be locked for writing // - lstores must all be locked for reading -func (s *store) imageTopLayerForMapping(image *Image, ristore roImageStore, primaryImageStore rwImageStore, rlstore rwLayerStore, lstores []roLayerStore, options types.IDMappingOptions) (*Layer, error) { +func (s *store) imageTopLayerForMapping(image *Image, ristore roImageStore, rlstore rwLayerStore, lstores []roLayerStore, options types.IDMappingOptions) (*Layer, error) { layerMatchesMappingOptions := func(layer *Layer, options types.IDMappingOptions) bool { // If the driver supports shifting and the layer has no mappings, we can use it. - if s.canUseShifting(options.UIDMap, options.GIDMap) && len(layer.UIDMap) == 0 && len(layer.GIDMap) == 0 { + if canUseShifting(rlstore, options.UIDMap, options.GIDMap) && len(layer.UIDMap) == 0 && len(layer.GIDMap) == 0 { return true } // If we want host mapping, and the layer uses mappings, it's not the best match. @@ -1334,7 +1354,7 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore roImageStore, prim var layer, parentLayer *Layer allStores := append([]roLayerStore{rlstore}, lstores...) // Locate the image's top layer and its parent, if it has one. - createMappedLayer := ristore == primaryImageStore + createMappedLayer := ristore == s.imageStore for _, s := range allStores { store := s // Walk the top layer list. @@ -1385,7 +1405,7 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore roImageStore, prim // that lets us edit image metadata, so create a duplicate of the layer with the desired // mappings, and register it as an alternate top layer in the image. var layerOptions LayerOptions - if s.canUseShifting(options.UIDMap, options.GIDMap) { + if canUseShifting(rlstore, options.UIDMap, options.GIDMap) { layerOptions = LayerOptions{ IDMappingOptions: types.IDMappingOptions{ HostUIDMapping: true, @@ -1409,8 +1429,8 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore roImageStore, prim if err != nil { return nil, fmt.Errorf("creating an ID-mapped copy of layer %q: %w", layer.ID, err) } - // By construction, createMappedLayer can only be true if ristore == primaryImageStore. - if err = primaryImageStore.addMappedTopLayer(image.ID, mappedLayer.ID); err != nil { + // By construction, createMappedLayer can only be true if ristore == s.imageStore. + if err = s.imageStore.addMappedTopLayer(image.ID, mappedLayer.ID); err != nil { if err2 := rlstore.Delete(mappedLayer.ID); err2 != nil { err = fmt.Errorf("deleting layer %q: %v: %w", mappedLayer.ID, err2, err) } @@ -1429,7 +1449,7 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat if options.HostGIDMapping { options.GIDMap = nil } - rlstore, err := s.getLayerStore() + rlstore, lstores, err := s.bothLayerStoreKinds() // lstores will be locked read-only if image != "" if err != nil { return nil, err } @@ -1447,24 +1467,10 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat } var imageHomeStore roImageStore // Set if image != "" - var istore rwImageStore // Set, and locked read-write, if image != "" - var istores []roImageStore // Set, and NOT NECESSARILY ALL locked read-only, if image != "" - var lstores []roLayerStore // Set, and locked read-only, if image != "" - var cimage *Image // Set if image != "" + // s.imageStore is locked read-write, if image != "" + // s.roImageStores are NOT NECESSARILY ALL locked read-only if image != "" + var cimage *Image // Set if image != "" if image != "" { - var err error - lstores, err = s.getROLayerStores() - if err != nil { - return nil, err - } - istore, err = s.getImageStore() - if err != nil { - return nil, err - } - istores, err = s.getROImageStores() - if err != nil { - return nil, err - } if err := rlstore.startWriting(); err != nil { return nil, err } @@ -1476,15 +1482,15 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat } defer store.stopReading() } - if err := istore.startWriting(); err != nil { + if err := s.imageStore.startWriting(); err != nil { return nil, err } - defer istore.stopWriting() - cimage, err = istore.Get(image) + defer s.imageStore.stopWriting() + cimage, err = s.imageStore.Get(image) if err == nil { - imageHomeStore = istore + imageHomeStore = s.imageStore } else { - for _, s := range istores { + for _, s := range s.roImageStores { store := s if err := store.startReading(); err != nil { return nil, err @@ -1517,7 +1523,7 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat idMappingsOptions := options.IDMappingOptions if image != "" { if cimage.TopLayer != "" { - ilayer, err := s.imageTopLayerForMapping(cimage, imageHomeStore, istore, rlstore, lstores, idMappingsOptions) + ilayer, err := s.imageTopLayerForMapping(cimage, imageHomeStore, rlstore, lstores, idMappingsOptions) if err != nil { return nil, err } @@ -1547,7 +1553,7 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat // But in transient store mode, all container layers are volatile. Volatile: options.Volatile || s.transientStore, } - if s.canUseShifting(uidMap, gidMap) { + if canUseShifting(rlstore, uidMap, gidMap) { layerOptions.IDMappingOptions = types.IDMappingOptions{ HostUIDMapping: true, @@ -1596,7 +1602,7 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat } var container *Container - err = s.writeToContainerStore(func(rcstore rwContainerStore) error { + err = s.writeToContainerStore(func() error { options.IDMappingOptions = types.IDMappingOptions{ HostUIDMapping: len(options.UIDMap) == 0, HostGIDMapping: len(options.GIDMap) == 0, @@ -1604,7 +1610,7 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat GIDMap: copyIDMap(options.GIDMap), } var err error - container, err = rcstore.Create(id, names, imageID, layer, metadata, options) + container, err = s.containerStore.Create(id, names, imageID, layer, metadata, options) if err != nil || container == nil { if err2 := rlstore.Delete(layer); err2 != nil { if err == nil { @@ -1620,15 +1626,15 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat } func (s *store) SetMetadata(id, metadata string) error { - return s.writeToAllStores(func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error { + return s.writeToAllStores(func(rlstore rwLayerStore) error { if rlstore.Exists(id) { return rlstore.SetMetadata(id, metadata) } - if ristore.Exists(id) { - return ristore.SetMetadata(id, metadata) + if s.imageStore.Exists(id) { + return s.imageStore.SetMetadata(id, metadata) } - if rcstore.Exists(id) { - return rcstore.SetMetadata(id, metadata) + if s.containerStore.Exists(id) { + return s.containerStore.SetMetadata(id, metadata) } return ErrNotAnID }) @@ -1659,16 +1665,12 @@ func (s *store) Metadata(id string) (string, error) { return res, err } - cstore, err := s.getContainerStore() - if err != nil { + if err := s.containerStore.startReading(); err != nil { return "", err } - if err := cstore.startReading(); err != nil { - return "", err - } - defer cstore.stopReading() - if cstore.Exists(id) { - return cstore.Metadata(id) + defer s.containerStore.stopReading() + if s.containerStore.Exists(id) { + return s.containerStore.Metadata(id) } return "", ErrNotAnID } @@ -1797,8 +1799,8 @@ func (s *store) SetLayerBigData(id, key string, data io.Reader) error { } func (s *store) SetImageBigData(id, key string, data []byte, digestManifest func([]byte) (digest.Digest, error)) error { - return s.writeToImageStore(func(ristore rwImageStore) error { - return ristore.SetBigData(id, key, data, digestManifest) + return s.writeToImageStore(func() error { + return s.imageStore.SetBigData(id, key, data, digestManifest) }) } @@ -1815,14 +1817,10 @@ func (s *store) ImageSize(id string) (int64, error) { defer store.stopReading() } - imageStores, err := s.allImageStores() - if err != nil { - return -1, err - } // Look for the image's record. var imageStore roBigDataStore var image *Image - for _, s := range imageStores { + for _, s := range s.allImageStores() { store := s if err := store.startReading(); err != nil { return -1, err @@ -1925,9 +1923,9 @@ func (s *store) ContainerSize(id string) (int64, error) { } var res int64 = -1 - err = s.writeToContainerStore(func(rcstore rwContainerStore) error { // Yes, rcstore.BigDataSize requires a write lock. + err = s.writeToContainerStore(func() error { // Yes, s.containerStore.BigDataSize requires a write lock. // Read the container record. - container, err := rcstore.Get(id) + container, err := s.containerStore.Get(id) if err != nil { return err } @@ -1949,12 +1947,12 @@ func (s *store) ContainerSize(id string) (int64, error) { } // Count big data items. - names, err := rcstore.BigDataNames(id) + names, err := s.containerStore.BigDataNames(id) if err != nil { return fmt.Errorf("reading list of big data items for container %q: %w", container.ID, err) } for _, name := range names { - n, err := rcstore.BigDataSize(id, name) + n, err := s.containerStore.BigDataSize(id, name) if err != nil { return fmt.Errorf("reading size of big data item %q for container %q: %w", name, id, err) } @@ -1980,24 +1978,19 @@ func (s *store) ContainerSize(id string) (int64, error) { } func (s *store) ListContainerBigData(id string) ([]string, error) { - rcstore, err := s.getContainerStore() - if err != nil { - return nil, err - } - - if err := rcstore.startReading(); err != nil { + if err := s.containerStore.startReading(); err != nil { return nil, err } - defer rcstore.stopReading() + defer s.containerStore.stopReading() - return rcstore.BigDataNames(id) + return s.containerStore.BigDataNames(id) } func (s *store) ContainerBigDataSize(id, key string) (int64, error) { var res int64 = -1 - err := s.writeToContainerStore(func(store rwContainerStore) error { // Yes, BigDataSize requires a write lock. + err := s.writeToContainerStore(func() error { // Yes, BigDataSize requires a write lock. var err error - res, err = store.BigDataSize(id, key) + res, err = s.containerStore.BigDataSize(id, key) return err }) return res, err @@ -2005,29 +1998,25 @@ func (s *store) ContainerBigDataSize(id, key string) (int64, error) { func (s *store) ContainerBigDataDigest(id, key string) (digest.Digest, error) { var res digest.Digest - err := s.writeToContainerStore(func(store rwContainerStore) error { // Yes, BigDataDigest requires a write lock. + err := s.writeToContainerStore(func() error { // Yes, BigDataDigest requires a write lock. var err error - res, err = store.BigDataDigest(id, key) + res, err = s.containerStore.BigDataDigest(id, key) return err }) return res, err } func (s *store) ContainerBigData(id, key string) ([]byte, error) { - rcstore, err := s.getContainerStore() - if err != nil { - return nil, err - } - if err := rcstore.startReading(); err != nil { + if err := s.containerStore.startReading(); err != nil { return nil, err } - defer rcstore.stopReading() - return rcstore.BigData(id, key) + defer s.containerStore.stopReading() + return s.containerStore.BigData(id, key) } func (s *store) SetContainerBigData(id, key string, data []byte) error { - return s.writeToContainerStore(func(rcstore rwContainerStore) error { - return rcstore.SetBigData(id, key, data) + return s.writeToContainerStore(func() error { + return s.containerStore.SetBigData(id, key, data) }) } @@ -2054,15 +2043,11 @@ func (s *store) Exists(id string) bool { return res } - rcstore, err := s.getContainerStore() - if err != nil { - return false - } - if err := rcstore.startReading(); err != nil { + if err := s.containerStore.startReading(); err != nil { return false } - defer rcstore.stopReading() - return rcstore.Exists(id) + defer s.containerStore.stopReading() + return s.containerStore.Exists(id) } func dedupeNames(names []string) []string { @@ -2104,25 +2089,17 @@ func (s *store) updateNames(id string, names []string, op updateNameOperation) e return err } - ristore, err := s.getImageStore() - if err != nil { + if err := s.imageStore.startWriting(); err != nil { return err } - if err := ristore.startWriting(); err != nil { - return err - } - defer ristore.stopWriting() - if ristore.Exists(id) { - return ristore.updateNames(id, deduped, op) + defer s.imageStore.stopWriting() + if s.imageStore.Exists(id) { + return s.imageStore.updateNames(id, deduped, op) } // Check is id refers to a RO Store - ristores, err := s.getROImageStores() - if err != nil { - return err - } - for _, s := range ristores { - store := s + for _, is := range s.roImageStores { + store := is if err := store.startReading(); err != nil { return err } @@ -2132,18 +2109,18 @@ func (s *store) updateNames(id string, names []string, op updateNameOperation) e // Do not want to create image name in R/W storage deduped = deduped[1:] } - _, err := ristore.Create(id, deduped, i.TopLayer, i.Metadata, i.Created, i.Digest) + _, err := s.imageStore.Create(id, deduped, i.TopLayer, i.Metadata, i.Created, i.Digest) return err } } containerFound := false - if err := s.writeToContainerStore(func(rcstore rwContainerStore) error { - if !rcstore.Exists(id) { + if err := s.writeToContainerStore(func() error { + if !s.containerStore.Exists(id) { return nil } containerFound = true - return rcstore.updateNames(id, deduped, op) + return s.containerStore.updateNames(id, deduped, op) }); err != nil || containerFound { return err } @@ -2174,15 +2151,11 @@ func (s *store) Names(id string) ([]string, error) { return res, err } - rcstore, err := s.getContainerStore() - if err != nil { - return nil, err - } - if err := rcstore.startReading(); err != nil { + if err := s.containerStore.startReading(); err != nil { return nil, err } - defer rcstore.stopReading() - if c, err := rcstore.Get(id); c != nil && err == nil { + defer s.containerStore.stopReading() + if c, err := s.containerStore.Get(id); c != nil && err == nil { return c.Names, nil } return nil, ErrLayerUnknown @@ -2211,15 +2184,11 @@ func (s *store) Lookup(name string) (string, error) { return res, err } - cstore, err := s.getContainerStore() - if err != nil { - return "", err - } - if err := cstore.startReading(); err != nil { + if err := s.containerStore.startReading(); err != nil { return "", err } - defer cstore.stopReading() - if c, err := cstore.Get(name); c != nil && err == nil { + defer s.containerStore.stopReading() + if c, err := s.containerStore.Get(name); c != nil && err == nil { return c.ID, nil } @@ -2227,7 +2196,7 @@ func (s *store) Lookup(name string) (string, error) { } func (s *store) DeleteLayer(id string) error { - return s.writeToAllStores(func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error { + return s.writeToAllStores(func(rlstore rwLayerStore) error { if rlstore.Exists(id) { if l, err := rlstore.Get(id); err != nil { id = l.ID @@ -2241,7 +2210,7 @@ func (s *store) DeleteLayer(id string) error { return fmt.Errorf("used by layer %v: %w", layer.ID, ErrLayerHasChildren) } } - images, err := ristore.Images() + images, err := s.imageStore.Images() if err != nil { return err } @@ -2251,7 +2220,7 @@ func (s *store) DeleteLayer(id string) error { return fmt.Errorf("layer %v used by image %v: %w", id, image.ID, ErrLayerUsedByImage) } } - containers, err := rcstore.Containers() + containers, err := s.containerStore.Containers() if err != nil { return err } @@ -2266,7 +2235,7 @@ func (s *store) DeleteLayer(id string) error { for _, image := range images { if stringutils.InSlice(image.MappedTopLayers, id) { - if err = ristore.removeMappedTopLayer(image.ID, id); err != nil { + if err = s.imageStore.removeMappedTopLayer(image.ID, id); err != nil { return fmt.Errorf("remove mapped top layer %v from image %v: %w", id, image.ID, err) } } @@ -2279,14 +2248,14 @@ func (s *store) DeleteLayer(id string) error { func (s *store) DeleteImage(id string, commit bool) (layers []string, err error) { layersToRemove := []string{} - if err := s.writeToAllStores(func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error { - if ristore.Exists(id) { - image, err := ristore.Get(id) + if err := s.writeToAllStores(func(rlstore rwLayerStore) error { + if s.imageStore.Exists(id) { + image, err := s.imageStore.Get(id) if err != nil { return err } id = image.ID - containers, err := rcstore.Containers() + containers, err := s.containerStore.Containers() if err != nil { return err } @@ -2297,7 +2266,7 @@ func (s *store) DeleteImage(id string, commit bool) (layers []string, err error) if container, ok := aContainerByImage[id]; ok { return fmt.Errorf("image used by %v: %w", container, ErrImageUsedByContainer) } - images, err := ristore.Images() + images, err := s.imageStore.Images() if err != nil { return err } @@ -2319,7 +2288,7 @@ func (s *store) DeleteImage(id string, commit bool) (layers []string, err error) } } if commit { - if err = ristore.Delete(id); err != nil { + if err = s.imageStore.Delete(id); err != nil { return err } } @@ -2330,7 +2299,7 @@ func (s *store) DeleteImage(id string, commit bool) (layers []string, err error) layersToRemoveMap[mappedTopLayer] = struct{}{} } for layer != "" { - if rcstore.Exists(layer) { + if s.containerStore.Exists(layer) { break } if _, used := otherImagesTopLayers[layer]; used { @@ -2382,12 +2351,12 @@ func (s *store) DeleteImage(id string, commit bool) (layers []string, err error) } func (s *store) DeleteContainer(id string) error { - return s.writeToAllStores(func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error { - if !rcstore.Exists(id) { + return s.writeToAllStores(func(rlstore rwLayerStore) error { + if !s.containerStore.Exists(id) { return ErrNotAContainer } - container, err := rcstore.Get(id) + container, err := s.containerStore.Get(id) if err != nil { return ErrNotAContainer } @@ -2404,7 +2373,7 @@ func (s *store) DeleteContainer(id string) error { } wg.Add(1) go func() { - errChan <- rcstore.Delete(id) + errChan <- s.containerStore.Delete(id) wg.Done() }() @@ -2453,14 +2422,14 @@ func (s *store) DeleteContainer(id string) error { } func (s *store) Delete(id string) error { - return s.writeToAllStores(func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error { - if rcstore.Exists(id) { - if container, err := rcstore.Get(id); err == nil { + return s.writeToAllStores(func(rlstore rwLayerStore) error { + if s.containerStore.Exists(id) { + if container, err := s.containerStore.Get(id); err == nil { if rlstore.Exists(container.LayerID) { if err = rlstore.Delete(container.LayerID); err != nil { return err } - if err = rcstore.Delete(id); err != nil { + if err = s.containerStore.Delete(id); err != nil { return err } middleDir := s.graphDriverName + "-containers" @@ -2477,8 +2446,8 @@ func (s *store) Delete(id string) error { return ErrNotALayer } } - if ristore.Exists(id) { - return ristore.Delete(id) + if s.imageStore.Exists(id) { + return s.imageStore.Delete(id) } if rlstore.Exists(id) { return rlstore.Delete(id) @@ -2488,11 +2457,11 @@ func (s *store) Delete(id string) error { } func (s *store) Wipe() error { - return s.writeToAllStores(func(rlstore rwLayerStore, ristore rwImageStore, rcstore rwContainerStore) error { - if err := rcstore.Wipe(); err != nil { + return s.writeToAllStores(func(rlstore rwLayerStore) error { + if err := s.containerStore.Wipe(); err != nil { return err } - if err := ristore.Wipe(); err != nil { + if err := s.imageStore.Wipe(); err != nil { return err } return rlstore.Wipe() @@ -2512,37 +2481,24 @@ func (s *store) Version() ([][2]string, error) { } func (s *store) mount(id string, options drivers.MountOpts) (string, error) { - rlstore, err := s.getLayerStore() - if err != nil { - return "", err - } - - s.graphLock.Lock() - defer s.graphLock.Unlock() - if err := rlstore.startWriting(); err != nil { + // We need to make sure the home mount is present when the Mount is done, which happens by possibly reinitializing the graph driver + // in startUsingGraphDriver(). + if err := s.startUsingGraphDriver(); err != nil { return "", err } - defer rlstore.stopWriting() + defer s.stopUsingGraphDriver() - lastWrite, modified, err := s.graphLock.ModifiedSince(s.graphLockLastWrite) + rlstore, err := s.getLayerStoreLocked() if err != nil { return "", err } - s.graphLockLastWrite = lastWrite - - /* We need to make sure the home mount is present when the Mount is done. */ - if modified { - s.graphDriver = nil - s.layerStore = nil - s.graphDriver, err = s.getGraphDriver() - if err != nil { - return "", err - } - s.lastLoaded = time.Now() + if err := rlstore.startWriting(); err != nil { + return "", err } + defer rlstore.stopWriting() if options.UidMaps != nil || options.GidMaps != nil { - options.DisableShifting = !s.canUseShifting(options.UidMaps, options.GidMaps) + options.DisableShifting = !canUseShifting(rlstore, options.UidMaps, options.GidMaps) } if rlstore.Exists(id) { @@ -2662,31 +2618,18 @@ func (s *store) DiffSize(from, to string) (int64, error) { } func (s *store) Diff(from, to string, options *DiffOptions) (io.ReadCloser, error) { - layerStores, err := s.allLayerStores() - if err != nil { - return nil, err - } - // NaiveDiff could cause mounts to happen without a lock, so be safe // and treat the .Diff operation as a Mount. - s.graphLock.Lock() - defer s.graphLock.Unlock() - - lastWrite, modified, err := s.graphLock.ModifiedSince(s.graphLockLastWrite) - if err != nil { + // We need to make sure the home mount is present when the Mount is done, which happens by possibly reinitializing the graph driver + // in startUsingGraphDriver(). + if err := s.startUsingGraphDriver(); err != nil { return nil, err } - s.graphLockLastWrite = lastWrite + defer s.stopUsingGraphDriver() - // We need to make sure the home mount is present when the Mount is done. - if modified { - s.graphDriver = nil - s.layerStore = nil - s.graphDriver, err = s.getGraphDriver() - if err != nil { - return nil, err - } - s.lastLoaded = time.Now() + layerStores, err := s.allLayerStoresLocked() + if err != nil { + return nil, err } for _, s := range layerStores { @@ -2836,19 +2779,15 @@ func (s *store) ContainerParentOwners(id string) ([]int, []int, error) { if err != nil { return nil, nil, err } - rcstore, err := s.getContainerStore() - if err != nil { - return nil, nil, err - } if err := rlstore.startReading(); err != nil { return nil, nil, err } defer rlstore.stopReading() - if err := rcstore.startReading(); err != nil { + if err := s.containerStore.startReading(); err != nil { return nil, nil, err } - defer rcstore.stopReading() - container, err := rcstore.Get(id) + defer s.containerStore.stopReading() + container, err := s.containerStore.Get(id) if err != nil { return nil, nil, err } @@ -2889,17 +2828,12 @@ func (s *store) Images() ([]Image, error) { } func (s *store) Containers() ([]Container, error) { - rcstore, err := s.getContainerStore() - if err != nil { - return nil, err - } - - if err := rcstore.startReading(); err != nil { + if err := s.containerStore.startReading(); err != nil { return nil, err } - defer rcstore.stopReading() + defer s.containerStore.stopReading() - return rcstore.Containers() + return s.containerStore.Containers() } func (s *store) Layer(id string) (*Layer, error) { @@ -2918,9 +2852,20 @@ func (s *store) Layer(id string) (*Layer, error) { } func (s *store) LookupAdditionalLayer(d digest.Digest, imageref string) (AdditionalLayer, error) { - adriver, ok := s.graphDriver.(drivers.AdditionalLayerStoreDriver) - if !ok { - return nil, ErrLayerUnknown + var adriver drivers.AdditionalLayerStoreDriver + if err := func() error { // A scope for defer + if err := s.startUsingGraphDriver(); err != nil { + return err + } + defer s.stopUsingGraphDriver() + a, ok := s.graphDriver.(drivers.AdditionalLayerStoreDriver) + if !ok { + return ErrLayerUnknown + } + adriver = a + return nil + }(); err != nil { + return nil, err } al, err := adriver.LookupAdditionalLayer(d, imageref) @@ -2957,7 +2902,7 @@ func (al *additionalLayer) CompressedSize() int64 { } func (al *additionalLayer) PutAs(id, parent string, names []string) (*Layer, error) { - rlstore, err := al.s.getLayerStore() + rlstore, rlstores, err := al.s.bothLayerStoreKinds() if err != nil { return nil, err } @@ -2965,10 +2910,6 @@ func (al *additionalLayer) PutAs(id, parent string, names []string) (*Layer, err return nil, err } defer rlstore.stopWriting() - rlstores, err := al.s.getROLayerStores() - if err != nil { - return nil, err - } var parentLayer *Layer if parent != "" { @@ -3052,28 +2993,20 @@ func (s *store) ImagesByDigest(d digest.Digest) ([]*Image, error) { } func (s *store) Container(id string) (*Container, error) { - rcstore, err := s.getContainerStore() - if err != nil { - return nil, err - } - if err := rcstore.startReading(); err != nil { + if err := s.containerStore.startReading(); err != nil { return nil, err } - defer rcstore.stopReading() + defer s.containerStore.stopReading() - return rcstore.Get(id) + return s.containerStore.Get(id) } func (s *store) ContainerLayerID(id string) (string, error) { - rcstore, err := s.getContainerStore() - if err != nil { - return "", err - } - if err := rcstore.startReading(); err != nil { + if err := s.containerStore.startReading(); err != nil { return "", err } - defer rcstore.stopReading() - container, err := rcstore.Get(id) + defer s.containerStore.stopReading() + container, err := s.containerStore.Get(id) if err != nil { return "", err } @@ -3085,15 +3018,11 @@ func (s *store) ContainerByLayer(id string) (*Container, error) { if err != nil { return nil, err } - rcstore, err := s.getContainerStore() - if err != nil { - return nil, err - } - if err := rcstore.startReading(); err != nil { + if err := s.containerStore.startReading(); err != nil { return nil, err } - defer rcstore.stopReading() - containerList, err := rcstore.Containers() + defer s.containerStore.stopReading() + containerList, err := s.containerStore.Containers() if err != nil { return nil, err } @@ -3107,16 +3036,12 @@ func (s *store) ContainerByLayer(id string) (*Container, error) { } func (s *store) ContainerDirectory(id string) (string, error) { - rcstore, err := s.getContainerStore() - if err != nil { - return "", err - } - if err := rcstore.startReading(); err != nil { + if err := s.containerStore.startReading(); err != nil { return "", err } - defer rcstore.stopReading() + defer s.containerStore.stopReading() - id, err = rcstore.Lookup(id) + id, err := s.containerStore.Lookup(id) if err != nil { return "", err } @@ -3130,17 +3055,12 @@ func (s *store) ContainerDirectory(id string) (string, error) { } func (s *store) ContainerRunDirectory(id string) (string, error) { - rcstore, err := s.getContainerStore() - if err != nil { - return "", err - } - - if err := rcstore.startReading(); err != nil { + if err := s.containerStore.startReading(); err != nil { return "", err } - defer rcstore.stopReading() + defer s.containerStore.stopReading() - id, err = rcstore.Lookup(id) + id, err := s.containerStore.Lookup(id) if err != nil { return "", err } @@ -3196,14 +3116,15 @@ func (s *store) FromContainerRunDirectory(id, file string) ([]byte, error) { func (s *store) Shutdown(force bool) ([]string, error) { mounted := []string{} - rlstore, err := s.getLayerStore() - if err != nil { + if err := s.startUsingGraphDriver(); err != nil { return mounted, err } + defer s.stopUsingGraphDriver() - s.graphLock.Lock() - defer s.graphLock.Unlock() - + rlstore, err := s.getLayerStoreLocked() + if err != nil { + return mounted, err + } if err := rlstore.startWriting(); err != nil { return nil, err } @@ -3393,8 +3314,8 @@ func (s *store) Free() { // Tries to clean up old unreferenced container leftovers. returns the first error // but continues as far as it can func (s *store) GarbageCollect() error { - firstErr := s.writeToContainerStore(func(rcstore rwContainerStore) error { - return rcstore.GarbageCollect() + firstErr := s.writeToContainerStore(func() error { + return s.containerStore.GarbageCollect() }) moreErr := s.writeToLayerStore(func(rlstore rwLayerStore) error { diff --git a/vendor/modules.txt b/vendor/modules.txt index de6333d95360..255e1c92e914 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -264,7 +264,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.44.1-0.20221201083122-c5a80ad65f42 +# github.com/containers/storage v1.44.1-0.20221201083122-c5a80ad65f42 => github.com/mtrmac/storage v0.0.0-20221201150117-26d820ccdc9e ## explicit; go 1.17 github.com/containers/storage github.com/containers/storage/drivers @@ -971,3 +971,4 @@ gopkg.in/yaml.v3 ## explicit; go 1.12 sigs.k8s.io/yaml # github.com/opencontainers/runc => github.com/opencontainers/runc v1.1.1-0.20220617142545-8b9452f75cbc +# github.com/containers/storage => github.com/mtrmac/storage v0.0.0-20221201150117-26d820ccdc9e