diff --git a/libcontainer/cgroups/systemd/common.go b/libcontainer/cgroups/systemd/common.go
index 53fa3f2c895..b567f3e1fcb 100644
--- a/libcontainer/cgroups/systemd/common.go
+++ b/libcontainer/cgroups/systemd/common.go
@@ -27,6 +27,9 @@ var (
 	versionOnce sync.Once
 	version     int
 	versionErr  error
+
+	isRunningSystemdOnce sync.Once
+	isRunningSystemd     bool
 )
 
 // NOTE: This function comes from package github.com/coreos/go-systemd/util
@@ -37,11 +40,11 @@ var (
 // checks whether /run/systemd/system/ exists and is a directory.
 // http://www.freedesktop.org/software/systemd/man/sd_booted.html
 func IsRunningSystemd() bool {
-	fi, err := os.Lstat("/run/systemd/system")
-	if err != nil {
-		return false
-	}
-	return fi.IsDir()
+	isRunningSystemdOnce.Do(func() {
+		fi, err := os.Lstat("/run/systemd/system")
+		isRunningSystemd = err == nil && fi.IsDir()
+	})
+	return isRunningSystemd
 }
 
 // systemd represents slice hierarchy using `-`, so we need to follow suit when
diff --git a/libcontainer/cgroups/systemd/v1.go b/libcontainer/cgroups/systemd/v1.go
index 0ce92636aa5..c3cc9c8b3bb 100644
--- a/libcontainer/cgroups/systemd/v1.go
+++ b/libcontainer/cgroups/systemd/v1.go
@@ -222,7 +222,14 @@ func (m *legacyManager) Destroy() error {
 		return err
 	}
 	unitName := getUnitName(m.cgroups)
-	if err := stopUnit(dbusConnection, unitName); err != nil {
+
+	err = stopUnit(dbusConnection, unitName)
+	// Both on success and on error, cleanup all the cgroups we are aware of.
+	// Some of them were created directly by Apply() and are not managed by systemd.
+	if err := cgroups.RemovePaths(m.paths); err != nil {
+		return err
+	}
+	if err != nil {
 		return err
 	}
 	m.paths = make(map[string]string)
diff --git a/tests/integration/delete.bats b/tests/integration/delete.bats
index bccd52e7245..b356e764758 100644
--- a/tests/integration/delete.bats
+++ b/tests/integration/delete.bats
@@ -12,24 +12,24 @@ function teardown() {
 }
 
 @test "runc delete" {
-  # run busybox detached
-  runc run -d --console-socket $CONSOLE_SOCKET test_busybox
+  runc run -d --console-socket $CONSOLE_SOCKET testbusyboxdelete
   [ "$status" -eq 0 ]
 
-  # check state
-  testcontainer test_busybox running
+  testcontainer testbusyboxdelete running
 
-  runc kill test_busybox KILL
+  runc kill testbusyboxdelete KILL
   [ "$status" -eq 0 ]
-  # wait for busybox to be in the destroyed state
-  retry 10 1 eval "__runc state test_busybox | grep -q 'stopped'"
+  retry 10 1 eval "__runc state testbusyboxdelete | grep -q 'stopped'"
 
-  # delete test_busybox
-  runc delete test_busybox
+  runc delete testbusyboxdelete
   [ "$status" -eq 0 ]
 
-  runc state test_busybox
+  runc state testbusyboxdelete
   [ "$status" -ne 0 ]
+
+  run find /sys/fs/cgroup -wholename '*testbusyboxdelete*' -type d
+  [ "$status" -eq 0 ]
+  [ "$output" = "" ] || fail "cgroup not cleaned up correctly: $output"
 }
 
 @test "runc delete --force" {