diff --git a/action/recipes/recipes.go b/action/recipes/recipes.go
index 830ccdfc..59f43219 100644
--- a/action/recipes/recipes.go
+++ b/action/recipes/recipes.go
@@ -7,6 +7,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
 	"os"
 	"path/filepath"
@@ -140,6 +141,25 @@ func Build(
 	return builds, err
 }
 
+// IsDirEmpty returns whether given directory is empty or not
+func IsDirEmpty(path string) (bool, error) {
+	// Source: https://stackoverflow.com/questions/30697324/how-to-check-if-directory-on-path-is-empty
+	f, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	defer f.Close()
+
+	// File.Readdirnames() take a parameter which is used to limit the number of returned values
+	// It is enough to query only 1 child
+	// File.Readdirnames() is faster than File.Readdir()
+	_, err = f.Readdirnames(1)
+	if err == io.EOF {
+		return true, nil
+	}
+	return false, err // Either not empty or error, suits both cases
+}
+
 // Execute a build step
 func Execute(ctx context.Context, target string, config *Config, interactive bool) error {
 	// Setup dagger client
@@ -152,8 +172,12 @@ func Execute(ctx context.Context, target string, config *Config, interactive boo
 	// Find requested target
 	modules := config.AllModules()
 	if _, ok := modules[target]; ok {
-		if _, err := os.Stat(modules[target].GetOutputDir()); err == nil {
 		// Check if output directory already exist
+		// We want to skip build if the output directory exists and is not empty
+		// If it is empty, then just continue with the building
+		_, errExists := os.Stat(modules[target].GetOutputDir())
+		empty, _ := IsDirEmpty(modules[target].GetOutputDir())
+		if errExists == nil && !empty {
 			slog.Warn(fmt.Sprintf("Output directory for '%s' already exists, skipping build", target))
 			return ErrBuildSkipped
 		}
diff --git a/action/recipes/recipes_test.go b/action/recipes/recipes_test.go
index 03388662..49f50c4e 100644
--- a/action/recipes/recipes_test.go
+++ b/action/recipes/recipes_test.go
@@ -4,6 +4,7 @@ package recipes
 import (
 	"context"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"dagger.io/dagger"
@@ -95,10 +96,18 @@ func TestExecuteSkipAndMissing(t *testing.T) {
 	assert.ErrorIs(t, err, ErrDependencyOutputMissing)
 
 	// Create the output directory
+	// Should build because the directory is empty
 	err = os.Mkdir(outputDir, os.ModePerm)
 	assert.NoError(t, err)
+	err = Execute(ctx, target, &myConfig, interactive)
+	assert.ErrorIs(t, err, ErrDependencyOutputMissing)
+
+	// Create file inside output directory
+	myfile, err := os.Create(filepath.Join(outputDir, "dummy.txt"))
+	assert.NoError(t, err)
+	myfile.Close()
 
-	// Since there is now existing output directory, it should skip the build
+	// Since there is now existing non-empty output directory, it should skip the build
 	err = Execute(ctx, target, &myConfig, interactive)
 	assert.ErrorIs(t, err, ErrBuildSkipped)
 }