diff --git a/imgconfig.go b/imgconfig.go
index b0c98c4d..056d25c4 100644
--- a/imgconfig.go
+++ b/imgconfig.go
@@ -36,12 +36,22 @@ func MergeSpecImage(spec *Spec, targetKey string) *ImageConfig {
 
 		cfg.Env = append(cfg.Env, i.Env...)
 
-		for k, v := range i.Volumes {
-			cfg.Volumes[k] = v
+		if len(i.Volumes) > 0 {
+			if cfg.Volumes == nil {
+				cfg.Volumes = make(map[string]struct{}, len(i.Volumes))
+			}
+			for k, v := range i.Volumes {
+				cfg.Volumes[k] = v
+			}
 		}
 
-		for k, v := range i.Labels {
-			cfg.Labels[k] = v
+		if len(i.Labels) > 0 {
+			if cfg.Labels == nil {
+				cfg.Labels = make(map[string]string, len(i.Labels))
+			}
+			for k, v := range i.Labels {
+				cfg.Labels[k] = v
+			}
 		}
 
 		if i.WorkingDir != "" {