From 8b3922112dc4aaaad26fe380192681d4795564f2 Mon Sep 17 00:00:00 2001 From: lburgazzoli Date: Fri, 3 Apr 2020 15:08:20 +0200 Subject: [PATCH 1/2] Include checksum when checking for compatible kits #743 --- examples/Sample.java | 2 +- pkg/apis/camel/v1/common_types.go | 1 + pkg/builder/builder_steps.go | 20 +++++++++++++------- pkg/builder/runtime/main.go | 15 +++++++++++++++ pkg/builder/runtime/quarkus.go | 25 +++++++++++++++++++++++++ pkg/controller/integrationkit/build.go | 1 + pkg/util/digest/digest.go | 22 ++++++++++++++++++++++ 7 files changed, 78 insertions(+), 8 deletions(-) diff --git a/examples/Sample.java b/examples/Sample.java index 0904fa4716..a8223acbc3 100644 --- a/examples/Sample.java +++ b/examples/Sample.java @@ -21,6 +21,6 @@ public class Sample extends RouteBuilder { @Override public void configure() throws Exception { from("timer:tick") - .log("Hello Camel K!"); + .log("Hello Camel K!"); } } \ No newline at end of file diff --git a/pkg/apis/camel/v1/common_types.go b/pkg/apis/camel/v1/common_types.go index f731aa8e36..7d45eeae71 100644 --- a/pkg/apis/camel/v1/common_types.go +++ b/pkg/apis/camel/v1/common_types.go @@ -33,6 +33,7 @@ type Artifact struct { ID string `json:"id" yaml:"id"` Location string `json:"location,omitempty" yaml:"location,omitempty"` Target string `json:"target,omitempty" yaml:"target,omitempty"` + Checksum string `json:"checksum,omitempty" yaml:"checksum,omitempty"` } // Failure -- diff --git a/pkg/builder/builder_steps.go b/pkg/builder/builder_steps.go index 0ecd89a4b3..cef2dd6dd7 100644 --- a/pkg/builder/builder_steps.go +++ b/pkg/builder/builder_steps.go @@ -368,9 +368,6 @@ func listPublishedImages(context *Context) ([]publishedImage, error) { for _, item := range list.Items { kit := item - if kit.Status.Phase != v1.IntegrationKitPhaseReady { - continue - } if kit.Status.Phase != v1.IntegrationKitPhaseReady { continue } @@ -391,9 +388,9 @@ func findBestImage(images []publishedImage, artifacts []v1.Artifact) (publishedI return bestImage, nil } - requiredLibs := make(map[string]bool, len(artifacts)) + requiredLibs := make(map[string]string, len(artifacts)) for _, entry := range artifacts { - requiredLibs[entry.ID] = true + requiredLibs[entry.ID] = entry.Checksum } bestImageCommonLibs := make(map[string]bool) @@ -402,7 +399,15 @@ func findBestImage(images []publishedImage, artifacts []v1.Artifact) (publishedI for _, image := range images { common := make(map[string]bool) for _, artifact := range image.Artifacts { - if _, ok := requiredLibs[artifact.ID]; ok { + // + // If the Artifact's checksum is not defined we can't reliably determine if for some + // reason the artifact has been changed but not the ID (as example for snapshots or + // other generated jar) thus we do not take this artifact into account. + // + if artifact.Checksum == "" { + continue + } + if requiredLibs[artifact.ID] == artifact.Checksum { common[artifact.ID] = true } } @@ -411,7 +416,8 @@ func findBestImage(images []publishedImage, artifacts []v1.Artifact) (publishedI surplus := len(image.Artifacts) - numCommonLibs if numCommonLibs != len(image.Artifacts) && surplus >= numCommonLibs/3 { - // Heuristic approach: if there are too many unrelated libraries, just use the base image + // Heuristic approach: if there are too many unrelated libraries then this image is + // not suitable to be used as base image continue } diff --git a/pkg/builder/runtime/main.go b/pkg/builder/runtime/main.go index 343cfbbeca..991f548827 100644 --- a/pkg/builder/runtime/main.go +++ b/pkg/builder/runtime/main.go @@ -22,6 +22,8 @@ import ( "io/ioutil" "path" + "github.com/apache/camel-k/pkg/util/digest" + "github.com/pkg/errors" yaml2 "gopkg.in/yaml.v2" @@ -117,10 +119,23 @@ func computeDependencies(ctx *builder.Context) error { return nil } + // + // Compute the checksum if it has not been computed by the camel-k-maven-plugin + // + if e.Checksum == "" { + chksum, err := digest.ComputeSHA1(e.Location) + if err != nil { + return err + } + + e.Checksum = "sha1:" + chksum + } + ctx.Artifacts = append(ctx.Artifacts, v1.Artifact{ ID: e.ID, Location: e.Location, Target: path.Join("dependencies", gav.GroupID+"."+fileName), + Checksum: e.Checksum, }) } diff --git a/pkg/builder/runtime/quarkus.go b/pkg/builder/runtime/quarkus.go index aefc706597..06655dc45c 100644 --- a/pkg/builder/runtime/quarkus.go +++ b/pkg/builder/runtime/quarkus.go @@ -23,6 +23,8 @@ import ( "os" "path" + "github.com/apache/camel-k/pkg/util/digest" + yaml2 "gopkg.in/yaml.v2" "github.com/pkg/errors" @@ -178,18 +180,41 @@ func computeQuarkusDependencies(ctx *builder.Context) error { return err } + // + // Compute the checksum if it has not been computed by the camel-k-maven-plugin + // + if e.Checksum == "" { + chksum, err := digest.ComputeSHA1(e.Location) + if err != nil { + return err + } + + e.Checksum = "sha1:" + chksum + } + ctx.Artifacts = append(ctx.Artifacts, v1.Artifact{ ID: e.ID, Location: e.Location, Target: path.Join("dependencies", gav.GroupID+"."+fileName), + Checksum: e.Checksum, }) } runner := "camel-k-integration-" + defaults.Version + "-runner.jar" + + // + // Quarkus' runner checksum need to be recomputed each time + // + runnerChecksum, err := digest.ComputeSHA1(mc.Path, "target", runner) + if err != nil { + return err + } + ctx.Artifacts = append(ctx.Artifacts, v1.Artifact{ ID: runner, Location: path.Join(mc.Path, "target", runner), Target: path.Join("dependencies", runner), + Checksum: "sha1:" + runnerChecksum, }) return nil diff --git a/pkg/controller/integrationkit/build.go b/pkg/controller/integrationkit/build.go index bed746f336..c182e3029d 100644 --- a/pkg/controller/integrationkit/build.go +++ b/pkg/controller/integrationkit/build.go @@ -166,6 +166,7 @@ func (action *buildAction) handleBuildRunning(ctx context.Context, kit *v1.Integ ID: a.ID, Location: "", Target: a.Target, + Checksum: a.Checksum, }) } diff --git a/pkg/util/digest/digest.go b/pkg/util/digest/digest.go index 53469f14f4..4c0b7cf1be 100644 --- a/pkg/util/digest/digest.go +++ b/pkg/util/digest/digest.go @@ -18,8 +18,12 @@ limitations under the License. package digest import ( + "crypto/sha1" "crypto/sha256" "encoding/base64" + "io" + "os" + "path" "sort" "strconv" @@ -167,3 +171,21 @@ func sortedTraitSpecMapKeys(m map[string]v1.TraitSpec) []string { sort.Strings(res) return res } + +// ComputeSHA1 --- +func ComputeSHA1(elem ...string) (string, error) { + file := path.Join(elem...) + + f, err := os.Open(file) + if err != nil { + return "", err + } + defer f.Close() + + h := sha1.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(h.Sum(nil)), nil +} From f9415abe354f646c5ddc10a758174226ab655691 Mon Sep 17 00:00:00 2001 From: lburgazzoli Date: Fri, 3 Apr 2020 15:17:45 +0200 Subject: [PATCH 2/2] Fix lint findings --- pkg/apis/camel/v1/common_types.go | 4 ++-- pkg/trait/container.go | 2 +- pkg/trait/knative.go | 4 ++-- pkg/trait/knative_test.go | 10 ++++++---- pkg/util/digest/digest.go | 2 ++ 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/apis/camel/v1/common_types.go b/pkg/apis/camel/v1/common_types.go index 7d45eeae71..6aecd5c871 100644 --- a/pkg/apis/camel/v1/common_types.go +++ b/pkg/apis/camel/v1/common_types.go @@ -113,8 +113,8 @@ const ( CapabilityHealth = "health" // CapabilityCron -- CapabilityCron = "cron" - // CapabilityPlatformHttp -- - CapabilityPlatformHttp = "platform-http" + // CapabilityPlatformHTTP -- + CapabilityPlatformHTTP = "platform-http" ) // ResourceCondition is a common type for all conditions diff --git a/pkg/trait/container.go b/pkg/trait/container.go index b5c4ec76e7..7f45e51860 100644 --- a/pkg/trait/container.go +++ b/pkg/trait/container.go @@ -419,7 +419,7 @@ func (t *containerTrait) configureCapabilities(e *Environment) error { requiresHTTP = true } - if util.StringSliceExists(e.Integration.Status.Capabilities, v1.CapabilityPlatformHttp) { + if util.StringSliceExists(e.Integration.Status.Capabilities, v1.CapabilityPlatformHTTP) { requiresHTTP = true } diff --git a/pkg/trait/knative.go b/pkg/trait/knative.go index 20747bb7b5..ef7fdb3a87 100644 --- a/pkg/trait/knative.go +++ b/pkg/trait/knative.go @@ -174,10 +174,10 @@ func (t *knativeTrait) Configure(e *Environment) (bool, error) { func (t *knativeTrait) Apply(e *Environment) error { if len(t.ChannelSources) > 0 || len(t.EndpointSources) > 0 || len(t.EventSources) > 0 { - util.StringSliceUniqueAdd(&e.Integration.Status.Capabilities, v1.CapabilityPlatformHttp) + util.StringSliceUniqueAdd(&e.Integration.Status.Capabilities, v1.CapabilityPlatformHTTP) } if len(t.ChannelSinks) > 0 || len(t.EndpointSinks) > 0 || len(t.EventSinks) > 0 { - util.StringSliceUniqueAdd(&e.Integration.Status.Capabilities, v1.CapabilityPlatformHttp) + util.StringSliceUniqueAdd(&e.Integration.Status.Capabilities, v1.CapabilityPlatformHTTP) } if e.IntegrationInPhase(v1.IntegrationPhaseDeploying, v1.IntegrationPhaseRunning) { diff --git a/pkg/trait/knative_test.go b/pkg/trait/knative_test.go index ad6b766bae..941e557454 100644 --- a/pkg/trait/knative_test.go +++ b/pkg/trait/knative_test.go @@ -264,7 +264,8 @@ func TestKnativePlatformHttpConfig(t *testing.T) { }, } - for _, source := range sources { + for _, ref := range sources { + source := ref t.Run(source.Name, func(t *testing.T) { environment := NewFakeEnvironment(t, source) @@ -279,7 +280,7 @@ func TestKnativePlatformHttpConfig(t *testing.T) { err = tc.apply(&environment) assert.Nil(t, err) - assert.Contains(t, environment.Integration.Status.Capabilities, v1.CapabilityPlatformHttp) + assert.Contains(t, environment.Integration.Status.Capabilities, v1.CapabilityPlatformHTTP) assert.Equal(t, "true", environment.ApplicationProperties["customizer.platform-http.enabled"]) assert.Equal(t, "8080", environment.ApplicationProperties["customizer.platform-http.bind-port"]) }) @@ -311,7 +312,8 @@ func TestKnativePlatformHttpDepdencies(t *testing.T) { }, } - for _, source := range sources { + for _, ref := range sources { + source := ref t.Run(source.Name, func(t *testing.T) { environment := NewFakeEnvironment(t, source) environment.Integration.Status.Phase = v1.IntegrationPhaseInitialization @@ -327,7 +329,7 @@ func TestKnativePlatformHttpDepdencies(t *testing.T) { err = tc.apply(&environment) assert.Nil(t, err) - assert.Contains(t, environment.Integration.Status.Capabilities, v1.CapabilityPlatformHttp) + assert.Contains(t, environment.Integration.Status.Capabilities, v1.CapabilityPlatformHTTP) assert.Contains(t, environment.Integration.Status.Dependencies, "mvn:org.apache.camel.k/camel-k-runtime-http") }) } diff --git a/pkg/util/digest/digest.go b/pkg/util/digest/digest.go index 4c0b7cf1be..c79f04584c 100644 --- a/pkg/util/digest/digest.go +++ b/pkg/util/digest/digest.go @@ -18,6 +18,7 @@ limitations under the License. package digest import ( + // nolint: gosec "crypto/sha1" "crypto/sha256" "encoding/base64" @@ -182,6 +183,7 @@ func ComputeSHA1(elem ...string) (string, error) { } defer f.Close() + // nolint: gosec h := sha1.New() if _, err := io.Copy(h, f); err != nil { return "", err