feat: directory content type with custom attributes (#390)

* Deprecate EmptyFolders and introduce dir contents.

* Handle explicit dirs with attributes correctly for Debs.

* Handle explicit dirs with attributes correctly for Apks.

* Sort contents by fields that are most relevant for the package.

* Make deprecation warning for empty_folders consistent.

* Name RPM directory tag more consistently.

* Fix deprecation notice in docs.

* Fix directory normalization for Debs and Apks.

* Revert Apk builder size in tests.

* Fix file sorting.

* Fix deprecated EmptyFolders handling in info.Validate.

* Fix Apk builder size in tests.

* Only stat content source when necessary.

* Allow src on dir content.

* Small addition to file_info docs.

* Fix typo in comments.

Co-authored-by: Carlos Alexandro Becker <caarlos0@users.noreply.github.com>

* Remove content type fallthrough case.

* Fix typo in apk tests.

* Fix more typos.

Co-authored-by: Carlos Alexandro Becker <caarlos0@users.noreply.github.com>

acceptance_test.go

@@ -189,6 +189,7 @@ func TestRPMSpecific(t *testing.T) {
 	format := "rpm"
 	testNames := []string{
 		"release",
+		"directories",
 	}
 	for _, name := range testNames {
 		for _, arch := range formatArchs[format] {

apk/apk.go

@@ -397,54 +397,65 @@ func createBuilderData(info *nfpm.Info, sizep *int64) func(tw *tar.Writer) error
 	created := map[string]bool{}
 
 	return func(tw *tar.Writer) error {
-		// handle empty folders
-		if err := createEmptyFoldersInsideTarGz(info, tw, created); err != nil {
-			return err
-		}
-
-		// handle Files
 		return createFilesInsideTarGz(info, tw, created, sizep)
 	}
 }
 
 func createFilesInsideTarGz(info *nfpm.Info, tw *tar.Writer, created map[string]bool, sizep *int64) (err error) {
+	// create explicit directories first
+	for _, file := range info.Contents {
+		// at this point, we don't care about other types yet
+		if file.Type != "dir" {
+			continue
+		}
+
+		// only consider contents for this packager
+		if file.Packager != "" && file.Packager != packagerName {
+			continue
+		}
+
+		err = tw.WriteHeader(&tar.Header{
+			Name:     files.ToNixPath(strings.Trim(file.Destination, "/") + "/"),
+			Mode:     int64(file.FileInfo.Mode),
+			Typeflag: tar.TypeDir,
+			Format:   tar.FormatGNU,
+			Uname:    file.FileInfo.Owner,
+			Gname:    file.FileInfo.Group,
+			ModTime:  file.FileInfo.MTime,
+		})
+		if err != nil {
+			return err
+		}
+
+		created[strings.TrimPrefix(file.Destination, "/")] = true
+	}
+
 	for _, file := range info.Contents {
+		// only consider contents for this packager
 		if file.Packager != "" && file.Packager != packagerName {
 			continue
 		}
+
+		// create implicit directory structure below the current content
 		if err = createTree(tw, file.Destination, created); err != nil {
 			return err
 		}
+
 		switch file.Type {
 		case "ghost":
 			// skip ghost files in apk
 			continue
+		case "dir":
+			// already handled above
+			continue
 		case "symlink":
 			err = createSymlinkInsideTarGz(file, tw)
-		case "doc":
-			// nolint:gocritic
-			// ignoring `emptyFallthrough: remove empty case containing only fallthrough to default case`
-			fallthrough
-		case "licence", "license":
-			// nolint:gocritic
-			// ignoring `emptyFallthrough: remove empty case containing only fallthrough to default case`
-			fallthrough
-		case "readme":
-			// nolint:gocritic
-			// ignoring `emptyFallthrough: remove empty case containing only fallthrough to default case`
-			fallthrough
-		case "config", "config|noreplace":
-			// nolint:gocritic
-			// ignoring `emptyFallthrough: remove empty case containing only fallthrough to default case`
-			fallthrough
 		default:
 			err = copyToTarAndDigest(file, tw, sizep)
 		}
 		if err != nil {
 			return err
 		}
-		created[file.Source] = true
-		created[file.Destination[1:]] = true
 	}
 
 	return nil
@@ -483,18 +494,6 @@ func copyToTarAndDigest(file *files.Content, tw *tar.Writer, sizep *int64) error
 	return nil
 }
 
-func createEmptyFoldersInsideTarGz(info *nfpm.Info, out *tar.Writer, created map[string]bool) error {
-	for _, folder := range info.EmptyFolders {
-		// this .nope is actually not created, because createTree ignore the
-		// last part of the path, assuming it is a file.
-		// TODO: should probably refactor this
-		if err := createTree(out, files.ToNixPath(filepath.Join(folder, ".nope")), created); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // this is needed because the data.tar.gz file should have the empty folders
 // as well, so we walk through the dst and create all subfolders.
 func createTree(tarw *tar.Writer, dst string, created map[string]bool) error {
@@ -518,8 +517,8 @@ func createTree(tarw *tar.Writer, dst string, created map[string]bool) error {
 }
 
 func pathsToCreate(dst string) []string {
-	var paths []string
-	base := dst[1:]
+	paths := []string{}
+	base := strings.Trim(dst, "/")
 	for {
 		base = filepath.Dir(base)
 		if base == "." {
@@ -529,7 +528,7 @@ func pathsToCreate(dst string) []string {
 	}
 	// we don't really need to create those things in order apparently, but,
 	// it looks really weird if we don't.
-	var result []string
+	result := []string{}
 	for i := len(paths) - 1; i >= 0; i-- {
 		result = append(result, paths[i])
 	}

apk/apk_test.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"path"
 	"path/filepath"
 	"testing"
 
@@ -74,10 +75,14 @@ func exampleInfo() *nfpm.Info {
 					Destination: "/etc/fake/fake.conf",
 					Type:        "config",
 				},
-			},
-			EmptyFolders: []string{
-				"/var/log/whatever",
-				"/usr/share/whatever",
+				{
+					Destination: "/var/log/whatever",
+					Type:        "dir",
+				},
+				{
+					Destination: "/usr/share/whatever",
+					Type:        "dir",
+				},
 			},
 		},
 	})
@@ -95,7 +100,7 @@ func TestCreateBuilderData(t *testing.T) {
 
 	require.NoError(t, builderData(tw))
 
-	require.Equal(t, 11784, buf.Len())
+	require.Equal(t, 12288, buf.Len())
 }
 
 func TestCombineToApk(t *testing.T) {
@@ -115,7 +120,7 @@ func TestPathsToCreate(t *testing.T) {
 	for pathToTest, parts := range map[string][]string{
 		"/usr/share/doc/whatever/foo.md": {"usr", "usr/share", "usr/share/doc", "usr/share/doc/whatever"},
 		"/var/moises":                    {"var"},
-		"/":                              []string(nil),
+		"/":                              {},
 	} {
 		parts := parts
 		pathToTest := pathToTest
@@ -462,6 +467,129 @@ func TestPackageSymlinks(t *testing.T) {
 	require.NoError(t, Default.Package(info, ioutil.Discard))
 }
 
+func TestDirectories(t *testing.T) {
+	info := exampleInfo()
+	info.Contents = []*files.Content{
+		{
+			Source:      "../testdata/whatever.conf",
+			Destination: "/etc/foo/file",
+		},
+		{
+			Source:      "../testdata/whatever.conf",
+			Destination: "/etc/bar/file",
+		},
+		{
+			Destination: "/etc/bar",
+			Type:        "dir",
+			FileInfo: &files.ContentFileInfo{
+				Owner: "test",
+				Mode:  0o700,
+			},
+		},
+		{
+			Destination: "/etc/baz",
+			Type:        "dir",
+		},
+	}
+
+	require.NoError(t, info.Validate())
+
+	var buf bytes.Buffer
+	size := int64(0)
+	err := createFilesInsideTarGz(info, tar.NewWriter(&buf), make(map[string]bool), &size)
+	require.NoError(t, err)
+
+	// for apks all implicit or explicit directories are created in the tarball
+	h := extractFileHeaderFromTar(t, buf.Bytes(), "/etc")
+	require.NoError(t, err)
+	require.Equal(t, h.Typeflag, byte(tar.TypeDir))
+	h = extractFileHeaderFromTar(t, buf.Bytes(), "/etc/foo")
+	require.NoError(t, err)
+	require.Equal(t, h.Typeflag, byte(tar.TypeDir))
+	h = extractFileHeaderFromTar(t, buf.Bytes(), "/etc/bar")
+	require.NoError(t, err)
+	require.Equal(t, h.Typeflag, byte(tar.TypeDir))
+	require.Equal(t, h.Mode, int64(0o700))
+	require.Equal(t, h.Uname, "test")
+	h = extractFileHeaderFromTar(t, buf.Bytes(), "/etc/baz")
+	require.NoError(t, err)
+	require.Equal(t, h.Typeflag, byte(tar.TypeDir))
+}
+
+func TestNoDuplicateContents(t *testing.T) {
+	info := exampleInfo()
+	info.Contents = []*files.Content{
+		{
+			Source:      "../testdata/whatever.conf",
+			Destination: "/etc/foo/file",
+		},
+		{
+			Source:      "../testdata/whatever.conf",
+			Destination: "/etc/bar/file",
+		},
+		{
+			Destination: "/etc/bar",
+			Type:        "dir",
+			FileInfo: &files.ContentFileInfo{
+				Owner: "test",
+				Mode:  0o700,
+			},
+		},
+		{
+			Destination: "/etc/baz",
+			Type:        "dir",
+		},
+	}
+
+	require.NoError(t, info.Validate())
+
+	var buf bytes.Buffer
+	size := int64(0)
+	err := createFilesInsideTarGz(info, tar.NewWriter(&buf), make(map[string]bool), &size)
+	require.NoError(t, err)
+
+	exists := map[string]bool{}
+
+	tr := tar.NewReader(bytes.NewReader(buf.Bytes()))
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			break // End of archive
+		}
+		require.NoError(t, err)
+
+		_, ok := exists[hdr.Name]
+		if ok {
+			t.Fatalf("%s exists more than once in tarball", hdr.Name)
+		}
+
+		exists[hdr.Name] = true
+	}
+}
+
+func extractFileHeaderFromTar(tb testing.TB, tarFile []byte, filename string) *tar.Header {
+	tb.Helper()
+
+	tr := tar.NewReader(bytes.NewReader(tarFile))
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			break // End of archive
+		}
+		require.NoError(tb, err)
+
+		if path.Join("/", hdr.Name) != path.Join("/", filename) { // nolint:gosec
+			continue
+		}
+
+		return hdr
+	}
+
+	tb.Fatalf("file %q does not exist in tar", filename)
+
+	return nil
+}
+
 func TestArches(t *testing.T) {
 	for k := range archToAlpine {
 		t.Run(k, func(t *testing.T) {