From 3df87c31aee3fb50c03fec0625fae6f3270ea7f8 Mon Sep 17 00:00:00 2001 From: AtomicFS Date: Wed, 4 Dec 2024 19:41:54 +0100 Subject: [PATCH] feat(action): add source changes detection - this should add the ability to detect changes in sources - in addition, the dagger setup was moved so now the loop for up-to-date module is much faster Signed-off-by: AtomicFS --- action/main.go | 4 +-- action/recipes/config.go | 15 ++++++++ action/recipes/recipes.go | 66 +++++++++++++++++++++++++++------- action/recipes/recipes_test.go | 10 ------ 4 files changed, 71 insertions(+), 24 deletions(-) diff --git a/action/main.go b/action/main.go index 92bd4844..4e7ed3b2 100644 --- a/action/main.go +++ b/action/main.go @@ -139,8 +139,8 @@ submodule_out: result := "" if item.BuildResult == nil { result = "Success" - } else if errors.Is(item.BuildResult, recipes.ErrBuildSkipped) { - result = "Skipped" + } else if errors.Is(item.BuildResult, recipes.ErrBuildUpToDate) { + result = "Up-to-date" } else { result = "Fail" } diff --git a/action/recipes/config.go b/action/recipes/config.go index 216dcbd9..9ee8ac85 100644 --- a/action/recipes/config.go +++ b/action/recipes/config.go @@ -155,6 +155,20 @@ func (opts CommonOpts) GetOutputDir() string { return opts.OutputDir } +// GetSources returns slice of paths to all sources which are used for build +func (opts CommonOpts) GetSources() []string { + sources := []string{} + + // Repository path + sources = append(sources, opts.RepoPath) + + // Input files and directories + sources = append(sources, opts.InputDirs[:]...) + sources = append(sources, opts.InputFiles[:]...) + + return sources +} + // Config is for storing parsed configuration file type Config struct { // defined in coreboot.go @@ -207,6 +221,7 @@ type FirmwareModule interface { GetContainerOutputDirs() []string GetContainerOutputFiles() []string GetOutputDir() string + GetSources() []string buildFirmware(ctx context.Context, client *dagger.Client, dockerfileDirectoryPath string) (*dagger.Container, error) } diff --git a/action/recipes/recipes.go b/action/recipes/recipes.go index 59f43219..39ac2715 100644 --- a/action/recipes/recipes.go +++ b/action/recipes/recipes.go @@ -16,13 +16,14 @@ import ( "dagger.io/dagger" "github.com/9elements/firmware-action/action/container" + "github.com/9elements/firmware-action/action/filesystem" "github.com/heimdalr/dag" ) // Errors for recipes var ( ErrBuildFailed = errors.New("build failed") - ErrBuildSkipped = errors.New("build skipped") + ErrBuildUpToDate = errors.New("build is up-to-date") ErrDependencyTreeUndefDep = errors.New("module has invalid dependency") ErrDependencyTreeUnderTarget = errors.New("target not found in dependency tree") ErrDependencyOutputMissing = errors.New("output of one or more dependencies is missing") @@ -31,8 +32,12 @@ var ( ErrTargetMissing = errors.New("no target specified") ) -// ContainerWorkDir specifies directory in container used as work directory -var ContainerWorkDir = "/workdir" +var ( + // ContainerWorkDir specifies directory in container used as work directory + ContainerWorkDir = "/workdir" + // TimestampsDir specifies directory for timestamps to detect changes in sources + TimestampsDir = ".firmware-action/timestamps" +) func forestAddVertex(forest *dag.DAG, key string, value FirmwareModule, dependencies [][]string) ([][]string, error) { err := forest.AddVertexByID(key, key) @@ -118,7 +123,7 @@ func Build( err = executor(ctx, item, config, interactive) builds = append(builds, BuildResults{item, err}) - if err != nil && !errors.Is(err, ErrBuildSkipped) { + if err != nil && !errors.Is(err, ErrBuildUpToDate) { break } } @@ -133,7 +138,7 @@ func Build( // Check results err = nil for _, item := range builds { - if item.BuildResult != nil && !errors.Is(item.BuildResult, ErrBuildSkipped) { + if item.BuildResult != nil && !errors.Is(item.BuildResult, ErrBuildUpToDate) { err = item.BuildResult } } @@ -161,25 +166,51 @@ func IsDirEmpty(path string) (bool, error) { } // Execute a build step +// func Execute(ctx context.Context, target string, config *Config, interactive bool, bulldozeMode bool) error { func Execute(ctx context.Context, target string, config *Config, interactive bool) error { - // Setup dagger client - client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stdout)) + // Prep + _, err := os.Stat(TimestampsDir) if err != nil { - return err + err = os.MkdirAll(TimestampsDir, os.ModePerm) + if err != nil { + return err + } } - defer client.Close() // Find requested target modules := config.AllModules() if _, ok := modules[target]; ok { + // Check if up-to-date + // Either returns time, or zero time and error + // zero time means there was no previous run + timestampFile := filepath.Join(TimestampsDir, fmt.Sprintf("%s.txt", target)) + lastRun, _ := filesystem.LoadLastRunTime(timestampFile) + + sources := modules[target].GetSources() + changesDetected := false + for _, source := range sources { + changes, _ := filesystem.AnyFileNewerThan(source, lastRun) + if changes { + changesDetected = true + break + } + } + // Check if output directory already exist // We want to skip build if the output directory exists and is not empty // If it is empty, then just continue with the building + // If changes in sources were detected, re-build _, errExists := os.Stat(modules[target].GetOutputDir()) empty, _ := IsDirEmpty(modules[target].GetOutputDir()) if errExists == nil && !empty { - slog.Warn(fmt.Sprintf("Output directory for '%s' already exists, skipping build", target)) - return ErrBuildSkipped + if changesDetected { + // If any of the sources changed, we need to rebuild + os.RemoveAll(modules[target].GetOutputDir()) + } else { + // Is already up-to-date + slog.Warn(fmt.Sprintf("Target '%s' is up-to-date, skipping build", target)) + return ErrBuildUpToDate + } } // Check if all outputs of required modules exist @@ -202,8 +233,19 @@ func Execute(ctx context.Context, target string, config *Config, interactive boo } } - // Build module + // Setup dagger client + client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stdout)) + if err != nil { + return err + } + defer client.Close() + + // Build the module myContainer, err := modules[target].buildFirmware(ctx, client, "") + if err == nil { + // On success update the timestamp + _ = filesystem.SaveCurrentRunTime(timestampFile) + } if err != nil && interactive { // If error, try to open SSH opts := container.NewSettingsSSH(container.WithWaitPressEnter()) diff --git a/action/recipes/recipes_test.go b/action/recipes/recipes_test.go index 49f50c4e..b72b87e1 100644 --- a/action/recipes/recipes_test.go +++ b/action/recipes/recipes_test.go @@ -4,7 +4,6 @@ package recipes import ( "context" "os" - "path/filepath" "testing" "dagger.io/dagger" @@ -101,15 +100,6 @@ func TestExecuteSkipAndMissing(t *testing.T) { assert.NoError(t, err) err = Execute(ctx, target, &myConfig, interactive) assert.ErrorIs(t, err, ErrDependencyOutputMissing) - - // Create file inside output directory - myfile, err := os.Create(filepath.Join(outputDir, "dummy.txt")) - assert.NoError(t, err) - myfile.Close() - - // Since there is now existing non-empty output directory, it should skip the build - err = Execute(ctx, target, &myConfig, interactive) - assert.ErrorIs(t, err, ErrBuildSkipped) } func executeDummy(_ context.Context, _ string, _ *Config, _ bool) error {