diff --git a/executor/oci/spec_linux.go b/executor/oci/spec_linux.go new file mode 100644 index 0000000000000..596b8e3cf65fd --- /dev/null +++ b/executor/oci/spec_linux.go @@ -0,0 +1,154 @@ +package oci + +import ( + "context" + "fmt" + "os" + "strings" + "sync" + + "github.com/containerd/containerd/containers" + "github.com/containerd/containerd/oci" + cdseccomp "github.com/containerd/containerd/pkg/seccomp" + "github.com/docker/docker/pkg/idtools" + "github.com/docker/docker/profiles/seccomp" + "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/entitlements/security" + specs "github.com/opencontainers/runtime-spec/specs-go" + selinux "github.com/opencontainers/selinux/go-selinux" + "github.com/opencontainers/selinux/go-selinux/label" + "github.com/pkg/errors" +) + +var ( + cgroupNSOnce sync.Once + supportsCgroupNS bool +) + +const ( + tracingSocketPath = "/dev/otel-grpc.sock" +) + +func generateMountOpts(resolvConf, hostsFile string) ([]oci.SpecOpts, error) { + return []oci.SpecOpts{ + // https://github.com/moby/buildkit/issues/429 + withRemovedMount("/run"), + withROBind(resolvConf, "/etc/resolv.conf"), + withROBind(hostsFile, "/etc/hosts"), + withCGroup(), + }, nil +} + +// generateSecurityOpts may affect mounts, so must be called after generateMountOpts +func generateSecurityOpts(mode pb.SecurityMode, apparmorProfile string, selinuxB bool) (opts []oci.SpecOpts, _ error) { + if selinuxB && !selinux.GetEnabled() { + return nil, errors.New("selinux is not available") + } + switch mode { + case pb.SecurityMode_INSECURE: + return []oci.SpecOpts{ + security.WithInsecureSpec(), + oci.WithWriteableCgroupfs, + oci.WithWriteableSysfs, + func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error { + var err error + if selinuxB { + s.Process.SelinuxLabel, s.Linux.MountLabel, err = label.InitLabels([]string{"disable"}) + } + return err + }, + }, nil + case pb.SecurityMode_SANDBOX: + if cdseccomp.IsEnabled() { + opts = append(opts, withDefaultProfile()) + } + if apparmorProfile != "" { + opts = append(opts, oci.WithApparmorProfile(apparmorProfile)) + } + opts = append(opts, func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error { + var err error + if selinuxB { + s.Process.SelinuxLabel, s.Linux.MountLabel, err = label.InitLabels(nil) + } + return err + }) + return opts, nil + } + return nil, nil +} + +// generateProcessModeOpts may affect mounts, so must be called after generateMountOpts +func generateProcessModeOpts(mode ProcessMode) ([]oci.SpecOpts, error) { + if mode == NoProcessSandbox { + return []oci.SpecOpts{ + oci.WithHostNamespace(specs.PIDNamespace), + withBoundProc(), + }, nil + // TODO(AkihiroSuda): Configure seccomp to disable ptrace (and prctl?) explicitly + } + return nil, nil +} + +func generateIDmapOpts(idmap *idtools.IdentityMapping) ([]oci.SpecOpts, error) { + if idmap == nil { + return nil, nil + } + return []oci.SpecOpts{ + oci.WithUserNamespace(specMapping(idmap.UIDMaps), specMapping(idmap.GIDMaps)), + }, nil +} + +func generateRlimitOpts(ulimits []*pb.Ulimit) ([]oci.SpecOpts, error) { + if len(ulimits) == 0 { + return nil, nil + } + var rlimits []specs.POSIXRlimit + for _, u := range ulimits { + if u == nil { + continue + } + rlimits = append(rlimits, specs.POSIXRlimit{ + Type: fmt.Sprintf("RLIMIT_%s", strings.ToUpper(u.Name)), + Hard: uint64(u.Hard), + Soft: uint64(u.Soft), + }) + } + return []oci.SpecOpts{ + func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { + s.Process.Rlimits = rlimits + return nil + }, + }, nil +} + +// withDefaultProfile sets the default seccomp profile to the spec. +// Note: must follow the setting of process capabilities +func withDefaultProfile() oci.SpecOpts { + return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { + var err error + s.Linux.Seccomp, err = seccomp.GetDefaultProfile(s) + return err + } +} + +func getTracingSocketMount(socket string) specs.Mount { + return specs.Mount{ + Destination: tracingSocketPath, + Type: "bind", + Source: socket, + Options: []string{"ro", "rbind"}, + } +} + +func getTracingSocket() string { + return fmt.Sprintf("unix://%s", tracingSocketPath) +} + +func cgroupNamespaceSupported() bool { + cgroupNSOnce.Do(func() { + if _, err := os.Stat("/proc/self/ns/cgroup"); !os.IsNotExist(err) { + supportsCgroupNS = true + } + }) + return supportsCgroupNS +} diff --git a/executor/oci/spec_unix.go b/executor/oci/spec_unix.go index 97e95e9834b27..bd2787bff2037 100644 --- a/executor/oci/spec_unix.go +++ b/executor/oci/spec_unix.go @@ -1,31 +1,17 @@ -//go:build !windows -// +build !windows +//go:build !linux && !windows package oci import ( "context" "fmt" - "os" "strings" - "sync" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/oci" - cdseccomp "github.com/containerd/containerd/pkg/seccomp" "github.com/docker/docker/pkg/idtools" - "github.com/docker/docker/profiles/seccomp" "github.com/moby/buildkit/solver/pb" - "github.com/moby/buildkit/util/entitlements/security" specs "github.com/opencontainers/runtime-spec/specs-go" - selinux "github.com/opencontainers/selinux/go-selinux" - "github.com/opencontainers/selinux/go-selinux/label" - "github.com/pkg/errors" -) - -var ( - cgroupNSOnce sync.Once - supportsCgroupNS bool ) const ( @@ -33,72 +19,19 @@ const ( ) func generateMountOpts(resolvConf, hostsFile string) ([]oci.SpecOpts, error) { - return []oci.SpecOpts{ - // https://github.com/moby/buildkit/issues/429 - withRemovedMount("/run"), - withROBind(resolvConf, "/etc/resolv.conf"), - withROBind(hostsFile, "/etc/hosts"), - withCGroup(), - }, nil + return nil, nil } -// generateSecurityOpts may affect mounts, so must be called after generateMountOpts func generateSecurityOpts(mode pb.SecurityMode, apparmorProfile string, selinuxB bool) (opts []oci.SpecOpts, _ error) { - if selinuxB && !selinux.GetEnabled() { - return nil, errors.New("selinux is not available") - } - switch mode { - case pb.SecurityMode_INSECURE: - return []oci.SpecOpts{ - security.WithInsecureSpec(), - oci.WithWriteableCgroupfs, - oci.WithWriteableSysfs, - func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error { - var err error - if selinuxB { - s.Process.SelinuxLabel, s.Linux.MountLabel, err = label.InitLabels([]string{"disable"}) - } - return err - }, - }, nil - case pb.SecurityMode_SANDBOX: - if cdseccomp.IsEnabled() { - opts = append(opts, withDefaultProfile()) - } - if apparmorProfile != "" { - opts = append(opts, oci.WithApparmorProfile(apparmorProfile)) - } - opts = append(opts, func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error { - var err error - if selinuxB { - s.Process.SelinuxLabel, s.Linux.MountLabel, err = label.InitLabels(nil) - } - return err - }) - return opts, nil - } return nil, nil } -// generateProcessModeOpts may affect mounts, so must be called after generateMountOpts func generateProcessModeOpts(mode ProcessMode) ([]oci.SpecOpts, error) { - if mode == NoProcessSandbox { - return []oci.SpecOpts{ - oci.WithHostNamespace(specs.PIDNamespace), - withBoundProc(), - }, nil - // TODO(AkihiroSuda): Configure seccomp to disable ptrace (and prctl?) explicitly - } return nil, nil } func generateIDmapOpts(idmap *idtools.IdentityMapping) ([]oci.SpecOpts, error) { - if idmap == nil { - return nil, nil - } - return []oci.SpecOpts{ - oci.WithUserNamespace(specMapping(idmap.UIDMaps), specMapping(idmap.GIDMaps)), - }, nil + return nil, nil } func generateRlimitOpts(ulimits []*pb.Ulimit) ([]oci.SpecOpts, error) { @@ -124,16 +57,6 @@ func generateRlimitOpts(ulimits []*pb.Ulimit) ([]oci.SpecOpts, error) { }, nil } -// withDefaultProfile sets the default seccomp profile to the spec. -// Note: must follow the setting of process capabilities -func withDefaultProfile() oci.SpecOpts { - return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error { - var err error - s.Linux.Seccomp, err = seccomp.GetDefaultProfile(s) - return err - } -} - func getTracingSocketMount(socket string) specs.Mount { return specs.Mount{ Destination: tracingSocketPath, @@ -148,10 +71,5 @@ func getTracingSocket() string { } func cgroupNamespaceSupported() bool { - cgroupNSOnce.Do(func() { - if _, err := os.Stat("/proc/self/ns/cgroup"); !os.IsNotExist(err) { - supportsCgroupNS = true - } - }) - return supportsCgroupNS + return false } diff --git a/snapshot/diffapply_unix.go b/snapshot/diffapply_unix.go index c4875000ea9e6..f88db2281b22f 100644 --- a/snapshot/diffapply_unix.go +++ b/snapshot/diffapply_unix.go @@ -11,16 +11,13 @@ import ( "strings" "syscall" - "github.com/containerd/containerd/leases" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/snapshots" - "github.com/containerd/containerd/snapshots/overlay/overlayutils" "github.com/containerd/continuity/fs" "github.com/containerd/continuity/sysx" "github.com/hashicorp/go-multierror" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/util/bklog" - "github.com/moby/buildkit/util/leaseutil" "github.com/moby/buildkit/util/overlay" "github.com/pkg/errors" "golang.org/x/sys/unix" @@ -128,7 +125,7 @@ func statInode(stat *syscall.Stat_t) inode { } return inode{ ino: stat.Ino, - dev: stat.Dev, + dev: uint64(stat.Dev), } } @@ -297,7 +294,7 @@ func (a *applier) applyDelete(ctx context.Context, ca *changeApply) (bool, error if ca.srcStat == nil { ca.srcStat = &syscall.Stat_t{ Mode: syscall.S_IFCHR, - Rdev: unix.Mkdev(0, 0), + Rdev: int32(unix.Mkdev(0, 0)), } ca.srcPath = "" } @@ -359,7 +356,7 @@ func (a *applier) applyCopy(ctx context.Context, ca *changeApply) error { case unix.S_IFDIR: if ca.dstStat == nil { // dstPath doesn't exist, make it a dir - if err := unix.Mkdir(ca.dstPath, ca.srcStat.Mode); err != nil { + if err := unix.Mkdir(ca.dstPath, uint32(ca.srcStat.Mode)); err != nil { return errors.Wrapf(err, "failed to create applied dir at %q from %q", ca.dstPath, ca.srcPath) } } @@ -370,7 +367,7 @@ func (a *applier) applyCopy(ctx context.Context, ca *changeApply) error { return errors.Wrap(err, "failed to create symlink during apply") } case unix.S_IFBLK, unix.S_IFCHR, unix.S_IFIFO, unix.S_IFSOCK: - if err := unix.Mknod(ca.dstPath, ca.srcStat.Mode, int(ca.srcStat.Rdev)); err != nil { + if err := unix.Mknod(ca.dstPath, uint32(ca.srcStat.Mode), int(ca.srcStat.Rdev)); err != nil { return errors.Wrap(err, "failed to mknod during apply") } default: @@ -385,7 +382,7 @@ func (a *applier) applyCopy(ctx context.Context, ca *changeApply) error { } if ca.srcStat.Mode&unix.S_IFMT != unix.S_IFLNK { - if err := unix.Chmod(ca.dstPath, ca.srcStat.Mode); err != nil { + if err := unix.Chmod(ca.dstPath, uint32(ca.srcStat.Mode)); err != nil { return errors.Wrapf(err, "failed to chmod path %q during apply", ca.dstPath) } } @@ -421,8 +418,8 @@ func (a *applier) applyCopy(ctx context.Context, ca *changeApply) error { } } - atimeSpec := unix.Timespec{Sec: ca.srcStat.Atim.Sec, Nsec: ca.srcStat.Atim.Nsec} - mtimeSpec := unix.Timespec{Sec: ca.srcStat.Mtim.Sec, Nsec: ca.srcStat.Mtim.Nsec} + atimeSpec := unix.Timespec(fs.StatAtime(ca.srcStat)) + mtimeSpec := unix.Timespec(fs.StatMtime(ca.srcStat)) if ca.srcStat.Mode&unix.S_IFMT != unix.S_IFDIR { // apply times immediately for non-dirs if err := unix.UtimesNanoAt(unix.AT_FDCWD, ca.dstPath, []unix.Timespec{atimeSpec, mtimeSpec}, unix.AT_SYMLINK_NOFOLLOW); err != nil { @@ -449,7 +446,7 @@ func (a *applier) Flush() error { return nil } if mtime, ok := a.dirModTimes[path]; ok { - if err := unix.UtimesNanoAt(unix.AT_FDCWD, path, []unix.Timespec{{Nsec: unix.UTIME_OMIT}, mtime}, unix.AT_SYMLINK_NOFOLLOW); err != nil { + if err := unix.UtimesNanoAt(unix.AT_FDCWD, path, []unix.Timespec{{Nsec: utimeOmit}, mtime}, unix.AT_SYMLINK_NOFOLLOW); err != nil { return err } } @@ -811,42 +808,3 @@ func opaqueXattr(userxattr bool) string { } return trustedOpaqueXattr } - -// needsUserXAttr checks whether overlay mounts should be provided the userxattr option. We can't use -// NeedsUserXAttr from the overlayutils package directly because we don't always have direct knowledge -// of the root of the snapshotter state (such as when using a remote snapshotter). Instead, we create -// a temporary new snapshot and test using its root, which works because single layer snapshots will -// use bind-mounts even when created by an overlay based snapshotter. -func needsUserXAttr(ctx context.Context, sn Snapshotter, lm leases.Manager) (bool, error) { - key := identity.NewID() - - ctx, done, err := leaseutil.WithLease(ctx, lm, leaseutil.MakeTemporary) - if err != nil { - return false, errors.Wrap(err, "failed to create lease for checking user xattr") - } - defer done(context.TODO()) - - err = sn.Prepare(ctx, key, "") - if err != nil { - return false, err - } - mntable, err := sn.Mounts(ctx, key) - if err != nil { - return false, err - } - mnts, unmount, err := mntable.Mount() - if err != nil { - return false, err - } - defer unmount() - - var userxattr bool - if err := mount.WithTempMount(ctx, mnts, func(root string) error { - var err error - userxattr, err = overlayutils.NeedsUserXAttr(root) - return err - }); err != nil { - return false, err - } - return userxattr, nil -} diff --git a/snapshot/diffapply_windows.go b/snapshot/diffapply_windows.go index 2ec9fc3d789cf..17fefae3b7c66 100644 --- a/snapshot/diffapply_windows.go +++ b/snapshot/diffapply_windows.go @@ -6,7 +6,6 @@ package snapshot import ( "context" - "github.com/containerd/containerd/leases" "github.com/containerd/containerd/snapshots" "github.com/pkg/errors" ) @@ -14,7 +13,3 @@ import ( func (sn *mergeSnapshotter) diffApply(ctx context.Context, dest Mountable, diffs ...Diff) (_ snapshots.Usage, rerr error) { return snapshots.Usage{}, errors.New("diffApply not yet supported on windows") } - -func needsUserXAttr(ctx context.Context, sn Snapshotter, lm leases.Manager) (bool, error) { - return false, errors.New("needs userxattr not supported on windows") -} diff --git a/snapshot/localmounter_unix.go b/snapshot/localmounter_unix.go index a4b7b1a9e4091..46c79d01274a5 100644 --- a/snapshot/localmounter_unix.go +++ b/snapshot/localmounter_unix.go @@ -5,7 +5,6 @@ package snapshot import ( "os" - "syscall" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/pkg/userns" @@ -65,7 +64,7 @@ func (lm *localMounter) Unmount() error { defer lm.mu.Unlock() if lm.target != "" { - if err := mount.Unmount(lm.target, syscall.MNT_DETACH); err != nil { + if err := mount.Unmount(lm.target, mntDetach); err != nil { return err } os.RemoveAll(lm.target) diff --git a/snapshot/mount_linux.go b/snapshot/mount_linux.go new file mode 100644 index 0000000000000..5b7384fec5b0e --- /dev/null +++ b/snapshot/mount_linux.go @@ -0,0 +1,7 @@ +package snapshot + +import ( + "golang.org/x/sys/unix" +) + +const mntDetach = unix.MNT_DETACH diff --git a/snapshot/mount_unix.go b/snapshot/mount_unix.go new file mode 100644 index 0000000000000..1da58092e2997 --- /dev/null +++ b/snapshot/mount_unix.go @@ -0,0 +1,5 @@ +//go:build !windows && !linux + +package snapshot + +const mntDetach = 0 diff --git a/snapshot/userxattr_linux.go b/snapshot/userxattr_linux.go new file mode 100644 index 0000000000000..bd80b357442d8 --- /dev/null +++ b/snapshot/userxattr_linux.go @@ -0,0 +1,40 @@ +package snapshot + +// needsUserXAttr checks whether overlay mounts should be provided the userxattr option. We can't use +// NeedsUserXAttr from the overlayutils package directly because we don't always have direct knowledge +// of the root of the snapshotter state (such as when using a remote snapshotter). Instead, we create +// a temporary new snapshot and test using its root, which works because single layer snapshots will +// use bind-mounts even when created by an overlay based snapshotter. +func needsUserXAttr(ctx context.Context, sn Snapshotter, lm leases.Manager) (bool, error) { + key := identity.NewID() + + ctx, done, err := leaseutil.WithLease(ctx, lm, leaseutil.MakeTemporary) + if err != nil { + return false, errors.Wrap(err, "failed to create lease for checking user xattr") + } + defer done(context.TODO()) + + err = sn.Prepare(ctx, key, "") + if err != nil { + return false, err + } + mntable, err := sn.Mounts(ctx, key) + if err != nil { + return false, err + } + mnts, unmount, err := mntable.Mount() + if err != nil { + return false, err + } + defer unmount() + + var userxattr bool + if err := mount.WithTempMount(ctx, mnts, func(root string) error { + var err error + userxattr, err = overlayutils.NeedsUserXAttr(root) + return err + }); err != nil { + return false, err + } + return userxattr, nil +} diff --git a/snapshot/userxattr_unsupported.go b/snapshot/userxattr_unsupported.go new file mode 100644 index 0000000000000..0399610b36d56 --- /dev/null +++ b/snapshot/userxattr_unsupported.go @@ -0,0 +1,13 @@ +//go:build !linux + +package snapshot + +import ( + "context" + "errors" + "github.com/containerd/containerd/leases" +) + +func needsUserXAttr(ctx context.Context, sn Snapshotter, lm leases.Manager) (bool, error) { + return false, errors.New("needs userxattr not supported on current OS") +} diff --git a/snapshot/utime_darwin.go b/snapshot/utime_darwin.go new file mode 100644 index 0000000000000..faef3f34aa258 --- /dev/null +++ b/snapshot/utime_darwin.go @@ -0,0 +1,4 @@ +package snapshot + +// macOS supports UTIME_OMIT starting with 10.13, but Go doesn't know about it yet. +const utimeOmit int64 = -2 diff --git a/snapshot/utime_unix.go b/snapshot/utime_unix.go new file mode 100644 index 0000000000000..0d7c72fa0cf4a --- /dev/null +++ b/snapshot/utime_unix.go @@ -0,0 +1,5 @@ +//go:build !darwin && !windows + +package snapshot + +const utimeOmit = unix.UTIME_OMIT diff --git a/util/overlay/overlay_linux.go b/util/overlay/overlay_linux.go index 62179f9ce825b..388eb0ac590c9 100644 --- a/util/overlay/overlay_linux.go +++ b/util/overlay/overlay_linux.go @@ -4,112 +4,18 @@ package overlay import ( - "bytes" "context" "fmt" "io" "os" - "path/filepath" "strings" - "sync" - "syscall" "time" "github.com/containerd/containerd/archive" "github.com/containerd/containerd/mount" - "github.com/containerd/continuity/devices" - "github.com/containerd/continuity/fs" - "github.com/containerd/continuity/sysx" "github.com/pkg/errors" - "golang.org/x/sys/unix" ) -// GetUpperdir parses the passed mounts and identifies the directory -// that contains diff between upper and lower. -func GetUpperdir(lower, upper []mount.Mount) (string, error) { - var upperdir string - if len(lower) == 0 && len(upper) == 1 { // upper is the bottommost snapshot - // Get layer directories of upper snapshot - upperM := upper[0] - if upperM.Type != "bind" { - return "", errors.Errorf("bottommost upper must be bind mount but %q", upperM.Type) - } - upperdir = upperM.Source - } else if len(lower) == 1 && len(upper) == 1 { - // Get layer directories of lower snapshot - var lowerlayers []string - lowerM := lower[0] - if lowerM.Type == "bind" { - // lower snapshot is a bind mount of one layer - lowerlayers = []string{lowerM.Source} - } else if IsOverlayMountType(lowerM) { - // lower snapshot is an overlay mount of multiple layers - var err error - lowerlayers, err = GetOverlayLayers(lowerM) - if err != nil { - return "", err - } - } else { - return "", errors.Errorf("cannot get layer information from mount option (type = %q)", lowerM.Type) - } - - // Get layer directories of upper snapshot - upperM := upper[0] - if !IsOverlayMountType(upperM) { - return "", errors.Errorf("upper snapshot isn't overlay mounted (type = %q)", upperM.Type) - } - upperlayers, err := GetOverlayLayers(upperM) - if err != nil { - return "", err - } - - // Check if the diff directory can be determined - if len(upperlayers) != len(lowerlayers)+1 { - return "", errors.Errorf("cannot determine diff of more than one upper directories") - } - for i := 0; i < len(lowerlayers); i++ { - if upperlayers[i] != lowerlayers[i] { - return "", errors.Errorf("layer %d must be common between upper and lower snapshots", i) - } - } - upperdir = upperlayers[len(upperlayers)-1] // get the topmost layer that indicates diff - } else { - return "", errors.Errorf("multiple mount configurations are not supported") - } - if upperdir == "" { - return "", errors.Errorf("cannot determine upperdir from mount option") - } - return upperdir, nil -} - -// GetOverlayLayers returns all layer directories of an overlayfs mount. -func GetOverlayLayers(m mount.Mount) ([]string, error) { - var u string - var uFound bool - var l []string // l[0] = bottommost - for _, o := range m.Options { - if strings.HasPrefix(o, "upperdir=") { - u, uFound = strings.TrimPrefix(o, "upperdir="), true - } else if strings.HasPrefix(o, "lowerdir=") { - l = strings.Split(strings.TrimPrefix(o, "lowerdir="), ":") - for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 { - l[i], l[j] = l[j], l[i] // make l[0] = bottommost - } - } else if strings.HasPrefix(o, "workdir=") || o == "index=off" || o == "userxattr" || strings.HasPrefix(o, "redirect_dir=") { - // these options are possible to specfied by the snapshotter but not indicate dir locations. - continue - } else { - // encountering an unknown option. return error and fallback to walking differ - // to avoid unexpected diff. - return nil, errors.Errorf("unknown option %q specified by snapshotter", o) - } - } - if uFound { - return append(l, u), nil - } - return l, nil -} - // WriteUpperdir writes a layer tar archive into the specified writer, based on // the diff information stored in the upperdir. func WriteUpperdir(ctx context.Context, w io.Writer, upperdir string, lower []mount.Mount) error { @@ -151,318 +57,3 @@ func (w *cancellableWriter) Write(p []byte) (int, error) { } return w.w.Write(p) } - -// Changes is continuty's `fs.Change`-like method but leverages overlayfs's -// "upperdir" for computing the diff. "upperdirView" is overlayfs mounted view of -// the upperdir that doesn't contain whiteouts. This is used for computing -// changes under opaque directories. -func Changes(ctx context.Context, changeFn fs.ChangeFunc, upperdir, upperdirView, base string) error { - return filepath.Walk(upperdir, func(path string, f os.FileInfo, err error) error { - if err != nil { - return err - } - if ctx.Err() != nil { - return ctx.Err() - } - - // Rebase path - path, err = filepath.Rel(upperdir, path) - if err != nil { - return err - } - path = filepath.Join(string(os.PathSeparator), path) - - // Skip root - if path == string(os.PathSeparator) { - return nil - } - - // Check redirect - if redirect, err := checkRedirect(upperdir, path, f); err != nil { - return err - } else if redirect { - // Return error when redirect_dir is enabled which can result to a wrong diff. - // TODO: support redirect_dir - return errors.New("redirect_dir is used but it's not supported in overlayfs differ") - } - - // Check if this is a deleted entry - isDelete, skip, err := checkDelete(upperdir, path, base, f) - if err != nil { - return err - } else if skip { - return nil - } - - var kind fs.ChangeKind - var skipRecord bool - if isDelete { - // This is a deleted entry. - kind = fs.ChangeKindDelete - // Leave f set to the FileInfo for the whiteout device in case the caller wants it, e.g. - // the merge code uses it to hardlink in the whiteout device to merged snapshots - } else if baseF, err := os.Lstat(filepath.Join(base, path)); err == nil { - // File exists in the base layer. Thus this is modified. - kind = fs.ChangeKindModify - // Avoid including directory that hasn't been modified. If /foo/bar/baz is modified, - // then /foo will apper here even if it's not been modified because it's the parent of bar. - if same, err := sameDirent(baseF, f, filepath.Join(base, path), filepath.Join(upperdirView, path)); same { - skipRecord = true // Both are the same, don't record the change - } else if err != nil { - return err - } - } else if os.IsNotExist(err) || errors.Is(err, unix.ENOTDIR) { - // File doesn't exist in the base layer. Thus this is added. - kind = fs.ChangeKindAdd - } else if err != nil { - return errors.Wrap(err, "failed to stat base file during overlay diff") - } - - if !skipRecord { - if err := changeFn(kind, path, f, nil); err != nil { - return err - } - } - - if f != nil { - if isOpaque, err := checkOpaque(upperdir, path, base, f); err != nil { - return err - } else if isOpaque { - // This is an opaque directory. Start a new walking differ to get adds/deletes of - // this directory. We use "upperdirView" directory which doesn't contain whiteouts. - if err := fs.Changes(ctx, filepath.Join(base, path), filepath.Join(upperdirView, path), - func(k fs.ChangeKind, p string, f os.FileInfo, err error) error { - return changeFn(k, filepath.Join(path, p), f, err) // rebase path to be based on the opaque dir - }, - ); err != nil { - return err - } - return filepath.SkipDir // We completed this directory. Do not walk files under this directory anymore. - } - } - return nil - }) -} - -// checkDelete checks if the specified file is a whiteout -func checkDelete(upperdir string, path string, base string, f os.FileInfo) (delete, skip bool, _ error) { - if f.Mode()&os.ModeCharDevice != 0 { - if _, ok := f.Sys().(*syscall.Stat_t); ok { - maj, min, err := devices.DeviceInfo(f) - if err != nil { - return false, false, errors.Wrapf(err, "failed to get device info") - } - if maj == 0 && min == 0 { - // This file is a whiteout (char 0/0) that indicates this is deleted from the base - if _, err := os.Lstat(filepath.Join(base, path)); err != nil { - if !os.IsNotExist(err) { - return false, false, errors.Wrapf(err, "failed to lstat") - } - // This file doesn't exist even in the base dir. - // We don't need whiteout. Just skip this file. - return false, true, nil - } - return true, false, nil - } - } - } - return false, false, nil -} - -// checkDelete checks if the specified file is an opaque directory -func checkOpaque(upperdir string, path string, base string, f os.FileInfo) (isOpaque bool, _ error) { - if f.IsDir() { - for _, oKey := range []string{"trusted.overlay.opaque", "user.overlay.opaque"} { - opaque, err := sysx.LGetxattr(filepath.Join(upperdir, path), oKey) - if err != nil && err != unix.ENODATA { - return false, errors.Wrapf(err, "failed to retrieve %s attr", oKey) - } else if len(opaque) == 1 && opaque[0] == 'y' { - // This is an opaque whiteout directory. - if _, err := os.Lstat(filepath.Join(base, path)); err != nil { - if !os.IsNotExist(err) { - return false, errors.Wrapf(err, "failed to lstat") - } - // This file doesn't exist even in the base dir. We don't need treat this as an opaque. - return false, nil - } - return true, nil - } - } - } - return false, nil -} - -// checkRedirect checks if the specified path enables redirect_dir. -func checkRedirect(upperdir string, path string, f os.FileInfo) (bool, error) { - if f.IsDir() { - rKey := "trusted.overlay.redirect" - redirect, err := sysx.LGetxattr(filepath.Join(upperdir, path), rKey) - if err != nil && err != unix.ENODATA { - return false, errors.Wrapf(err, "failed to retrieve %s attr", rKey) - } - return len(redirect) > 0, nil - } - return false, nil -} - -// sameDirent performs continity-compatible comparison of files and directories. -// https://github.com/containerd/continuity/blob/v0.1.0/fs/path.go#L91-L133 -// This will only do a slow content comparison of two files if they have all the -// same metadata and both have truncated nanosecond mtime timestamps. In practice, -// this can only happen if both the base file in the lowerdirs has a truncated -// timestamp (i.e. was unpacked from a tar) and the user did something like -// "mv foo tmp && mv tmp foo" that results in the file being copied up to the -// upperdir without making any changes to it. This is much rarer than similar -// cases in the double-walking differ, where the slow content comparison will -// be used whenever a file with a truncated timestamp is in the lowerdir at -// all and left unmodified. -func sameDirent(f1, f2 os.FileInfo, f1fullPath, f2fullPath string) (bool, error) { - if os.SameFile(f1, f2) { - return true, nil - } - - equalStat, err := compareSysStat(f1.Sys(), f2.Sys()) - if err != nil || !equalStat { - return equalStat, err - } - - if eq, err := compareCapabilities(f1fullPath, f2fullPath); err != nil || !eq { - return eq, err - } - - if !f1.IsDir() { - if f1.Size() != f2.Size() { - return false, nil - } - t1 := f1.ModTime() - t2 := f2.ModTime() - - if t1.Unix() != t2.Unix() { - return false, nil - } - - // If the timestamp may have been truncated in both of the - // files, check content of file to determine difference - if t1.Nanosecond() == 0 && t2.Nanosecond() == 0 { - if (f1.Mode() & os.ModeSymlink) == os.ModeSymlink { - return compareSymlinkTarget(f1fullPath, f2fullPath) - } - if f1.Size() == 0 { - return true, nil - } - return compareFileContent(f1fullPath, f2fullPath) - } else if t1.Nanosecond() != t2.Nanosecond() { - return false, nil - } - } - - return true, nil -} - -// Ported from continuity project -// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_unix.go#L43-L54 -// Copyright The containerd Authors. -func compareSysStat(s1, s2 interface{}) (bool, error) { - ls1, ok := s1.(*syscall.Stat_t) - if !ok { - return false, nil - } - ls2, ok := s2.(*syscall.Stat_t) - if !ok { - return false, nil - } - - return ls1.Mode == ls2.Mode && ls1.Uid == ls2.Uid && ls1.Gid == ls2.Gid && ls1.Rdev == ls2.Rdev, nil -} - -// Ported from continuity project -// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_unix.go#L56-L66 -// Copyright The containerd Authors. -func compareCapabilities(p1, p2 string) (bool, error) { - c1, err := sysx.LGetxattr(p1, "security.capability") - if err != nil && err != sysx.ENODATA { - return false, errors.Wrapf(err, "failed to get xattr for %s", p1) - } - c2, err := sysx.LGetxattr(p2, "security.capability") - if err != nil && err != sysx.ENODATA { - return false, errors.Wrapf(err, "failed to get xattr for %s", p2) - } - return bytes.Equal(c1, c2), nil -} - -// Ported from continuity project -// https://github.com/containerd/continuity/blob/bce1c3f9669b6f3e7f6656ee715b0b4d75fa64a6/fs/path.go#L135 -// Copyright The containerd Authors. -func compareSymlinkTarget(p1, p2 string) (bool, error) { - t1, err := os.Readlink(p1) - if err != nil { - return false, err - } - t2, err := os.Readlink(p2) - if err != nil { - return false, err - } - return t1 == t2, nil -} - -var bufPool = sync.Pool{ - New: func() interface{} { - b := make([]byte, 32*1024) - return &b - }, -} - -// Ported from continuity project -// https://github.com/containerd/continuity/blob/bce1c3f9669b6f3e7f6656ee715b0b4d75fa64a6/fs/path.go#L151 -// Copyright The containerd Authors. -func compareFileContent(p1, p2 string) (bool, error) { - f1, err := os.Open(p1) - if err != nil { - return false, err - } - defer f1.Close() - if stat, err := f1.Stat(); err != nil { - return false, err - } else if !stat.Mode().IsRegular() { - return false, errors.Errorf("%s is not a regular file", p1) - } - - f2, err := os.Open(p2) - if err != nil { - return false, err - } - defer f2.Close() - if stat, err := f2.Stat(); err != nil { - return false, err - } else if !stat.Mode().IsRegular() { - return false, errors.Errorf("%s is not a regular file", p2) - } - - b1 := bufPool.Get().(*[]byte) - defer bufPool.Put(b1) - b2 := bufPool.Get().(*[]byte) - defer bufPool.Put(b2) - for { - n1, err1 := io.ReadFull(f1, *b1) - if err1 == io.ErrUnexpectedEOF { - // it's expected to get EOF when file size isn't a multiple of chunk size, consolidate these error types - err1 = io.EOF - } - if err1 != nil && err1 != io.EOF { - return false, err1 - } - n2, err2 := io.ReadFull(f2, *b2) - if err2 == io.ErrUnexpectedEOF { - err2 = io.EOF - } - if err2 != nil && err2 != io.EOF { - return false, err2 - } - if n1 != n2 || !bytes.Equal((*b1)[:n1], (*b2)[:n2]) { - return false, nil - } - if err1 == io.EOF && err2 == io.EOF { - return true, nil - } - } -} diff --git a/util/overlay/overlay_unix.go b/util/overlay/overlay_unix.go new file mode 100644 index 0000000000000..8db6f7c7d98a5 --- /dev/null +++ b/util/overlay/overlay_unix.go @@ -0,0 +1,422 @@ +//go:build !windows + +package overlay + +import ( + "bytes" + "context" + "io" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + + "github.com/containerd/containerd/mount" + "github.com/containerd/continuity/devices" + "github.com/containerd/continuity/fs" + "github.com/containerd/continuity/sysx" + "github.com/pkg/errors" + "golang.org/x/sys/unix" +) + +// GetOverlayLayers returns all layer directories of an overlayfs mount. +func GetOverlayLayers(m mount.Mount) ([]string, error) { + var u string + var uFound bool + var l []string // l[0] = bottommost + for _, o := range m.Options { + if strings.HasPrefix(o, "upperdir=") { + u, uFound = strings.TrimPrefix(o, "upperdir="), true + } else if strings.HasPrefix(o, "lowerdir=") { + l = strings.Split(strings.TrimPrefix(o, "lowerdir="), ":") + for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 { + l[i], l[j] = l[j], l[i] // make l[0] = bottommost + } + } else if strings.HasPrefix(o, "workdir=") || o == "index=off" || o == "userxattr" || strings.HasPrefix(o, "redirect_dir=") { + // these options are possible to specfied by the snapshotter but not indicate dir locations. + continue + } else { + // encountering an unknown option. return error and fallback to walking differ + // to avoid unexpected diff. + return nil, errors.Errorf("unknown option %q specified by snapshotter", o) + } + } + if uFound { + return append(l, u), nil + } + return l, nil +} + +// GetUpperdir parses the passed mounts and identifies the directory +// that contains diff between upper and lower. +func GetUpperdir(lower, upper []mount.Mount) (string, error) { + var upperdir string + if len(lower) == 0 && len(upper) == 1 { // upper is the bottommost snapshot + // Get layer directories of upper snapshot + upperM := upper[0] + if upperM.Type != "bind" { + return "", errors.Errorf("bottommost upper must be bind mount but %q", upperM.Type) + } + upperdir = upperM.Source + } else if len(lower) == 1 && len(upper) == 1 { + // Get layer directories of lower snapshot + var lowerlayers []string + lowerM := lower[0] + if lowerM.Type == "bind" { + // lower snapshot is a bind mount of one layer + lowerlayers = []string{lowerM.Source} + } else if IsOverlayMountType(lowerM) { + // lower snapshot is an overlay mount of multiple layers + var err error + lowerlayers, err = GetOverlayLayers(lowerM) + if err != nil { + return "", err + } + } else { + return "", errors.Errorf("cannot get layer information from mount option (type = %q)", lowerM.Type) + } + + // Get layer directories of upper snapshot + upperM := upper[0] + if !IsOverlayMountType(upperM) { + return "", errors.Errorf("upper snapshot isn't overlay mounted (type = %q)", upperM.Type) + } + upperlayers, err := GetOverlayLayers(upperM) + if err != nil { + return "", err + } + + // Check if the diff directory can be determined + if len(upperlayers) != len(lowerlayers)+1 { + return "", errors.Errorf("cannot determine diff of more than one upper directories") + } + for i := 0; i < len(lowerlayers); i++ { + if upperlayers[i] != lowerlayers[i] { + return "", errors.Errorf("layer %d must be common between upper and lower snapshots", i) + } + } + upperdir = upperlayers[len(upperlayers)-1] // get the topmost layer that indicates diff + } else { + return "", errors.Errorf("multiple mount configurations are not supported") + } + if upperdir == "" { + return "", errors.Errorf("cannot determine upperdir from mount option") + } + return upperdir, nil +} + +// Changes is continuty's `fs.Change`-like method but leverages overlayfs's +// "upperdir" for computing the diff. "upperdirView" is overlayfs mounted view of +// the upperdir that doesn't contain whiteouts. This is used for computing +// changes under opaque directories. +func Changes(ctx context.Context, changeFn fs.ChangeFunc, upperdir, upperdirView, base string) error { + return filepath.Walk(upperdir, func(path string, f os.FileInfo, err error) error { + if err != nil { + return err + } + if ctx.Err() != nil { + return ctx.Err() + } + + // Rebase path + path, err = filepath.Rel(upperdir, path) + if err != nil { + return err + } + path = filepath.Join(string(os.PathSeparator), path) + + // Skip root + if path == string(os.PathSeparator) { + return nil + } + + // Check redirect + if redirect, err := checkRedirect(upperdir, path, f); err != nil { + return err + } else if redirect { + // Return error when redirect_dir is enabled which can result to a wrong diff. + // TODO: support redirect_dir + return errors.New("redirect_dir is used but it's not supported in overlayfs differ") + } + + // Check if this is a deleted entry + isDelete, skip, err := checkDelete(upperdir, path, base, f) + if err != nil { + return err + } else if skip { + return nil + } + + var kind fs.ChangeKind + var skipRecord bool + if isDelete { + // This is a deleted entry. + kind = fs.ChangeKindDelete + // Leave f set to the FileInfo for the whiteout device in case the caller wants it, e.g. + // the merge code uses it to hardlink in the whiteout device to merged snapshots + } else if baseF, err := os.Lstat(filepath.Join(base, path)); err == nil { + // File exists in the base layer. Thus this is modified. + kind = fs.ChangeKindModify + // Avoid including directory that hasn't been modified. If /foo/bar/baz is modified, + // then /foo will apper here even if it's not been modified because it's the parent of bar. + if same, err := sameDirent(baseF, f, filepath.Join(base, path), filepath.Join(upperdirView, path)); same { + skipRecord = true // Both are the same, don't record the change + } else if err != nil { + return err + } + } else if os.IsNotExist(err) || errors.Is(err, unix.ENOTDIR) { + // File doesn't exist in the base layer. Thus this is added. + kind = fs.ChangeKindAdd + } else if err != nil { + return errors.Wrap(err, "failed to stat base file during overlay diff") + } + + if !skipRecord { + if err := changeFn(kind, path, f, nil); err != nil { + return err + } + } + + if f != nil { + if isOpaque, err := checkOpaque(upperdir, path, base, f); err != nil { + return err + } else if isOpaque { + // This is an opaque directory. Start a new walking differ to get adds/deletes of + // this directory. We use "upperdirView" directory which doesn't contain whiteouts. + if err := fs.Changes(ctx, filepath.Join(base, path), filepath.Join(upperdirView, path), + func(k fs.ChangeKind, p string, f os.FileInfo, err error) error { + return changeFn(k, filepath.Join(path, p), f, err) // rebase path to be based on the opaque dir + }, + ); err != nil { + return err + } + return filepath.SkipDir // We completed this directory. Do not walk files under this directory anymore. + } + } + return nil + }) +} + +// checkRedirect checks if the specified path enables redirect_dir. +func checkRedirect(upperdir string, path string, f os.FileInfo) (bool, error) { + if f.IsDir() { + rKey := "trusted.overlay.redirect" + redirect, err := sysx.LGetxattr(filepath.Join(upperdir, path), rKey) + if err != nil && err != unix.ENODATA { + return false, errors.Wrapf(err, "failed to retrieve %s attr", rKey) + } + return len(redirect) > 0, nil + } + return false, nil +} + +// checkDelete checks if the specified file is an opaque directory +func checkOpaque(upperdir string, path string, base string, f os.FileInfo) (isOpaque bool, _ error) { + if f.IsDir() { + for _, oKey := range []string{"trusted.overlay.opaque", "user.overlay.opaque"} { + opaque, err := sysx.LGetxattr(filepath.Join(upperdir, path), oKey) + if err != nil && err != unix.ENODATA { + return false, errors.Wrapf(err, "failed to retrieve %s attr", oKey) + } else if len(opaque) == 1 && opaque[0] == 'y' { + // This is an opaque whiteout directory. + if _, err := os.Lstat(filepath.Join(base, path)); err != nil { + if !os.IsNotExist(err) { + return false, errors.Wrapf(err, "failed to lstat") + } + // This file doesn't exist even in the base dir. We don't need treat this as an opaque. + return false, nil + } + return true, nil + } + } + } + return false, nil +} + +// checkDelete checks if the specified file is a whiteout +func checkDelete(upperdir string, path string, base string, f os.FileInfo) (delete, skip bool, _ error) { + if f.Mode()&os.ModeCharDevice != 0 { + if _, ok := f.Sys().(*syscall.Stat_t); ok { + maj, min, err := devices.DeviceInfo(f) + if err != nil { + return false, false, errors.Wrapf(err, "failed to get device info") + } + if maj == 0 && min == 0 { + // This file is a whiteout (char 0/0) that indicates this is deleted from the base + if _, err := os.Lstat(filepath.Join(base, path)); err != nil { + if !os.IsNotExist(err) { + return false, false, errors.Wrapf(err, "failed to lstat") + } + // This file doesn't exist even in the base dir. + // We don't need whiteout. Just skip this file. + return false, true, nil + } + return true, false, nil + } + } + } + return false, false, nil +} + +// sameDirent performs continity-compatible comparison of files and directories. +// https://github.com/containerd/continuity/blob/v0.1.0/fs/path.go#L91-L133 +// This will only do a slow content comparison of two files if they have all the +// same metadata and both have truncated nanosecond mtime timestamps. In practice, +// this can only happen if both the base file in the lowerdirs has a truncated +// timestamp (i.e. was unpacked from a tar) and the user did something like +// "mv foo tmp && mv tmp foo" that results in the file being copied up to the +// upperdir without making any changes to it. This is much rarer than similar +// cases in the double-walking differ, where the slow content comparison will +// be used whenever a file with a truncated timestamp is in the lowerdir at +// all and left unmodified. +func sameDirent(f1, f2 os.FileInfo, f1fullPath, f2fullPath string) (bool, error) { + if os.SameFile(f1, f2) { + return true, nil + } + + equalStat, err := compareSysStat(f1.Sys(), f2.Sys()) + if err != nil || !equalStat { + return equalStat, err + } + + if eq, err := compareCapabilities(f1fullPath, f2fullPath); err != nil || !eq { + return eq, err + } + + if !f1.IsDir() { + if f1.Size() != f2.Size() { + return false, nil + } + t1 := f1.ModTime() + t2 := f2.ModTime() + + if t1.Unix() != t2.Unix() { + return false, nil + } + + // If the timestamp may have been truncated in both of the + // files, check content of file to determine difference + if t1.Nanosecond() == 0 && t2.Nanosecond() == 0 { + if (f1.Mode() & os.ModeSymlink) == os.ModeSymlink { + return compareSymlinkTarget(f1fullPath, f2fullPath) + } + if f1.Size() == 0 { + return true, nil + } + return compareFileContent(f1fullPath, f2fullPath) + } else if t1.Nanosecond() != t2.Nanosecond() { + return false, nil + } + } + + return true, nil +} + +var bufPool = sync.Pool{ + New: func() interface{} { + b := make([]byte, 32*1024) + return &b + }, +} + +// Ported from continuity project +// https://github.com/containerd/continuity/blob/bce1c3f9669b6f3e7f6656ee715b0b4d75fa64a6/fs/path.go#L151 +// Copyright The containerd Authors. +func compareFileContent(p1, p2 string) (bool, error) { + f1, err := os.Open(p1) + if err != nil { + return false, err + } + defer f1.Close() + if stat, err := f1.Stat(); err != nil { + return false, err + } else if !stat.Mode().IsRegular() { + return false, errors.Errorf("%s is not a regular file", p1) + } + + f2, err := os.Open(p2) + if err != nil { + return false, err + } + defer f2.Close() + if stat, err := f2.Stat(); err != nil { + return false, err + } else if !stat.Mode().IsRegular() { + return false, errors.Errorf("%s is not a regular file", p2) + } + + b1 := bufPool.Get().(*[]byte) + defer bufPool.Put(b1) + b2 := bufPool.Get().(*[]byte) + defer bufPool.Put(b2) + for { + n1, err1 := io.ReadFull(f1, *b1) + if err1 == io.ErrUnexpectedEOF { + // it's expected to get EOF when file size isn't a multiple of chunk size, consolidate these error types + err1 = io.EOF + } + if err1 != nil && err1 != io.EOF { + return false, err1 + } + n2, err2 := io.ReadFull(f2, *b2) + if err2 == io.ErrUnexpectedEOF { + err2 = io.EOF + } + if err2 != nil && err2 != io.EOF { + return false, err2 + } + if n1 != n2 || !bytes.Equal((*b1)[:n1], (*b2)[:n2]) { + return false, nil + } + if err1 == io.EOF && err2 == io.EOF { + return true, nil + } + } +} + +// Ported from continuity project +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_unix.go#L43-L54 +// Copyright The containerd Authors. +func compareSysStat(s1, s2 interface{}) (bool, error) { + ls1, ok := s1.(*syscall.Stat_t) + if !ok { + return false, nil + } + ls2, ok := s2.(*syscall.Stat_t) + if !ok { + return false, nil + } + + return ls1.Mode == ls2.Mode && ls1.Uid == ls2.Uid && ls1.Gid == ls2.Gid && ls1.Rdev == ls2.Rdev, nil +} + +// Ported from continuity project +// https://github.com/containerd/continuity/blob/v0.1.0/fs/diff_unix.go#L56-L66 +// Copyright The containerd Authors. +func compareCapabilities(p1, p2 string) (bool, error) { + c1, err := sysx.LGetxattr(p1, "security.capability") + if err != nil && err != sysx.ENODATA { + return false, errors.Wrapf(err, "failed to get xattr for %s", p1) + } + c2, err := sysx.LGetxattr(p2, "security.capability") + if err != nil && err != sysx.ENODATA { + return false, errors.Wrapf(err, "failed to get xattr for %s", p2) + } + return bytes.Equal(c1, c2), nil +} + +// Ported from continuity project +// https://github.com/containerd/continuity/blob/bce1c3f9669b6f3e7f6656ee715b0b4d75fa64a6/fs/path.go#L135 +// Copyright The containerd Authors. +func compareSymlinkTarget(p1, p2 string) (bool, error) { + t1, err := os.Readlink(p1) + if err != nil { + return false, err + } + t2, err := os.Readlink(p2) + if err != nil { + return false, err + } + return t1 == t2, nil +}