From 2052f5465f9f74426d7956285275b0dba2ada3f3 Mon Sep 17 00:00:00 2001 From: lburgazzoli Date: Fri, 19 Jul 2019 18:13:44 +0200 Subject: [PATCH] fix build-pod strategy on kubernetes --- deploy/builder-role-kubernetes.yaml | 1 + deploy/resources.go | 1 + pkg/controller/build/build_controller.go | 3 +- pkg/controller/build/initialize_pod.go | 89 +++++++++++++++++++ .../{initialize.go => initialize_routine.go} | 17 ++-- pkg/controller/build/monitor_pod.go | 26 +++--- pkg/controller/build/schedule_pod.go | 70 ++++----------- pkg/controller/build/util_pod.go | 73 +++++++++++++++ 8 files changed, 204 insertions(+), 76 deletions(-) create mode 100644 pkg/controller/build/initialize_pod.go rename pkg/controller/build/{initialize.go => initialize_routine.go} (65%) create mode 100644 pkg/controller/build/util_pod.go diff --git a/deploy/builder-role-kubernetes.yaml b/deploy/builder-role-kubernetes.yaml index e3ee33eebe..db17ffd05d 100644 --- a/deploy/builder-role-kubernetes.yaml +++ b/deploy/builder-role-kubernetes.yaml @@ -28,6 +28,7 @@ rules: - "" resources: - events + - configmaps verbs: - get - list diff --git a/deploy/resources.go b/deploy/resources.go index ad80ee60bb..f4b51a7201 100644 --- a/deploy/resources.go +++ b/deploy/resources.go @@ -73,6 +73,7 @@ rules: - "" resources: - events + - configmaps verbs: - get - list diff --git a/pkg/controller/build/build_controller.go b/pkg/controller/build/build_controller.go index 1d6ca04863..3a011287d0 100644 --- a/pkg/controller/build/build_controller.go +++ b/pkg/controller/build/build_controller.go @@ -190,7 +190,8 @@ func (r *ReconcileBuild) Reconcile(request reconcile.Request) (reconcile.Result, } actions := []Action{ - NewInitializeAction(), + NewInitializeRoutineAction(), + NewInitializePodAction(), NewScheduleRoutineAction(r.reader, r.builder, &r.routines), NewSchedulePodAction(r.reader), NewMonitorRoutineAction(&r.routines), diff --git a/pkg/controller/build/initialize_pod.go b/pkg/controller/build/initialize_pod.go new file mode 100644 index 0000000000..45768728c5 --- /dev/null +++ b/pkg/controller/build/initialize_pod.go @@ -0,0 +1,89 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package build + +import ( + "context" + + "github.com/apache/camel-k/pkg/install" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" +) + +// NewInitializePodAction creates a new initialize action +func NewInitializePodAction() Action { + return &initializePodAction{} +} + +type initializePodAction struct { + baseAction +} + +// Name returns a common name of the action +func (action *initializePodAction) Name() string { + return "initialize-pod" +} + +// CanHandle tells whether this action can handle the build +func (action *initializePodAction) CanHandle(build *v1alpha1.Build) bool { + return build.Status.Phase == v1alpha1.BuildPhaseInitialization && + build.Spec.Platform.Build.BuildStrategy == v1alpha1.IntegrationPlatformBuildStrategyPod +} + +// Handle handles the builds +func (action *initializePodAction) Handle(ctx context.Context, build *v1alpha1.Build) (*v1alpha1.Build, error) { + // Ensure service account is present + // TODO: maybe this should be done by the platform trait ?? + if err := action.ensureServiceAccount(ctx, build); err != nil { + return nil, errors.Wrap(err, "cannot ensure service account is present") + } + + if err := deleteBuilderPod(ctx, action.client, build); err != nil { + return nil, errors.Wrap(err, "cannot delete build pod") + } + + pod, err := getBuilderPod(ctx, action.client, build) + if err != nil || pod != nil { + // We return and wait for the pod to be deleted before de-queue the build pod. + return nil, err + } + + build.Status.Phase = v1alpha1.BuildPhaseScheduling + + return build, nil +} + +func (action *initializePodAction) ensureServiceAccount(ctx context.Context, build *v1alpha1.Build) error { + sa := corev1.ServiceAccount{} + saKey := k8sclient.ObjectKey{ + Name: "camel-k-builder", + Namespace: build.Namespace, + } + + err := action.client.Get(ctx, saKey, &sa) + if err != nil && k8serrors.IsNotFound(err) { + // Create a proper service account + return install.BuilderServiceAccountRoles(ctx, action.client, build.Namespace) + } + + return err +} diff --git a/pkg/controller/build/initialize.go b/pkg/controller/build/initialize_routine.go similarity index 65% rename from pkg/controller/build/initialize.go rename to pkg/controller/build/initialize_routine.go index a3c0674601..0f37b033a4 100644 --- a/pkg/controller/build/initialize.go +++ b/pkg/controller/build/initialize_routine.go @@ -23,27 +23,28 @@ import ( "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" ) -// NewInitializeAction creates a new initialize action -func NewInitializeAction() Action { - return &initializeAction{} +// NewInitializeRoutineAction creates a new initialize action +func NewInitializeRoutineAction() Action { + return &initializeRoutineAction{} } -type initializeAction struct { +type initializeRoutineAction struct { baseAction } // Name returns a common name of the action -func (action *initializeAction) Name() string { +func (action *initializeRoutineAction) Name() string { return "initialize" } // CanHandle tells whether this action can handle the build -func (action *initializeAction) CanHandle(build *v1alpha1.Build) bool { - return build.Status.Phase == v1alpha1.BuildPhaseInitialization +func (action *initializeRoutineAction) CanHandle(build *v1alpha1.Build) bool { + return build.Status.Phase == v1alpha1.BuildPhaseInitialization && + build.Spec.Platform.Build.BuildStrategy == v1alpha1.IntegrationPlatformBuildStrategyRoutine } // Handle handles the builds -func (action *initializeAction) Handle(ctx context.Context, build *v1alpha1.Build) (*v1alpha1.Build, error) { +func (action *initializeRoutineAction) Handle(ctx context.Context, build *v1alpha1.Build) (*v1alpha1.Build, error) { build.Status.Phase = v1alpha1.BuildPhaseScheduling return build, nil diff --git a/pkg/controller/build/monitor_pod.go b/pkg/controller/build/monitor_pod.go index 7979e34033..3b7a185c9c 100644 --- a/pkg/controller/build/monitor_pod.go +++ b/pkg/controller/build/monitor_pod.go @@ -20,11 +20,8 @@ package build import ( "context" - corev1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" + corev1 "k8s.io/api/core/v1" ) // NewMonitorPodAction creates a new monitor action for scheduled pod @@ -51,24 +48,21 @@ func (action *monitorPodAction) CanHandle(build *v1alpha1.Build) bool { // Handle handles the builds func (action *monitorPodAction) Handle(ctx context.Context, build *v1alpha1.Build) (*v1alpha1.Build, error) { // Get the build pod - pod := &corev1.Pod{} - err := action.client.Get(ctx, types.NamespacedName{Namespace: build.Namespace, Name: buildPodName(build.Spec.Meta)}, pod) + pod, err := getBuilderPod(ctx, action.client, build) if err != nil { - if k8serrors.IsNotFound(err) { - // Let's reschedule the build - build.Status.Phase = v1alpha1.BuildPhaseScheduling - } else { - return nil, err - } + return nil, err } - var buildPhase v1alpha1.BuildPhase - switch pod.Status.Phase { - case corev1.PodSucceeded: + switch { + case pod == nil: + build.Status.Phase = v1alpha1.BuildPhaseScheduling + case pod.Status.Phase == corev1.PodSucceeded: buildPhase = v1alpha1.BuildPhaseSucceeded - case corev1.PodFailed: + case pod.Status.Phase == corev1.PodFailed: buildPhase = v1alpha1.BuildPhaseFailed + default: + buildPhase = build.Status.Phase } if build.Status.Phase == buildPhase { diff --git a/pkg/controller/build/schedule_pod.go b/pkg/controller/build/schedule_pod.go index a5f38da45d..2b719b1f75 100644 --- a/pkg/controller/build/schedule_pod.go +++ b/pkg/controller/build/schedule_pod.go @@ -22,12 +22,10 @@ import ( "sync" "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" - "github.com/apache/camel-k/pkg/install" "github.com/apache/camel-k/pkg/platform" "github.com/apache/camel-k/pkg/util/defaults" corev1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -77,48 +75,37 @@ func (action *schedulePodAction) Handle(ctx context.Context, build *v1alpha1.Bui // Emulate a serialized working queue to only allow one build to run at a given time. // This is currently necessary for the incremental build to work as expected. - hasScheduledBuild := false for _, b := range builds.Items { if b.Status.Phase == v1alpha1.BuildPhasePending || b.Status.Phase == v1alpha1.BuildPhaseRunning { - hasScheduledBuild = true - break + // Let's requeue the build in case one is already running + return nil, nil } } - if hasScheduledBuild { - // Let's requeue the build in case one is already running - return nil, nil - } - - // Try to get operator image name before starting the build - operatorImage, err := platform.GetCurrentOperatorImage(ctx, action.client) + pod, err := getBuilderPod(ctx, action.client, build) if err != nil { return nil, err } - // Otherwise, let's create the build pod - // We may want to explicitly manage build priority as opposed to relying on - // the reconcile loop to handle the queuing - pod := newBuildPod(build, operatorImage) - - // Set the Build instance as the owner and controller - if err := controllerutil.SetControllerReference(build, pod, action.client.GetScheme()); err != nil { - return nil, err - } + if pod == nil { + // Try to get operator image name before starting the build + operatorImage, err := platform.GetCurrentOperatorImage(ctx, action.client) + if err != nil { + return nil, err + } - // Ensure service account is present - if err := action.ensureServiceAccount(ctx, pod); err != nil { - return nil, errors.Wrap(err, "cannot ensure service account is present") - } + // We may want to explicitly manage build priority as opposed to relying on + // the reconcile loop to handle the queuing + pod = newBuildPod(build, operatorImage) - err = action.client.Delete(ctx, pod) - if err != nil && !k8serrors.IsNotFound(err) { - return nil, errors.Wrap(err, "cannot delete build pod") - } + // Set the Build instance as the owner and controller + if err := controllerutil.SetControllerReference(build, pod, action.client.GetScheme()); err != nil { + return nil, err + } - err = action.client.Create(ctx, pod) - if err != nil { - return nil, errors.Wrap(err, "cannot create build pod") + if err := action.client.Create(ctx, pod); err != nil { + return nil, errors.Wrap(err, "cannot create build pod") + } } build.Status.Phase = v1alpha1.BuildPhasePending @@ -126,21 +113,6 @@ func (action *schedulePodAction) Handle(ctx context.Context, build *v1alpha1.Bui return build, nil } -func (action *schedulePodAction) ensureServiceAccount(ctx context.Context, buildPod *corev1.Pod) error { - sa := corev1.ServiceAccount{} - saKey := k8sclient.ObjectKey{ - Name: "camel-k-builder", - Namespace: buildPod.Namespace, - } - - err := action.client.Get(ctx, saKey, &sa) - if err != nil && k8serrors.IsNotFound(err) { - // Create a proper service account - return install.BuilderServiceAccountRoles(ctx, action.client, buildPod.Namespace) - } - return err -} - func newBuildPod(build *v1alpha1.Build, operatorImage string) *corev1.Pod { builderImage := operatorImage if builderImage == "" { @@ -219,7 +191,3 @@ func newBuildPod(build *v1alpha1.Build, operatorImage string) *corev1.Pod { return pod } - -func buildPodName(object metav1.ObjectMeta) string { - return "camel-k-" + object.Name + "-builder" -} diff --git a/pkg/controller/build/util_pod.go b/pkg/controller/build/util_pod.go new file mode 100644 index 0000000000..6189e38549 --- /dev/null +++ b/pkg/controller/build/util_pod.go @@ -0,0 +1,73 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package build + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apache/camel-k/pkg/apis/camel/v1alpha1" +) + +// deleteBuilderPod -- +func deleteBuilderPod(ctx context.Context, client k8sclient.Writer, build *v1alpha1.Build) error { + pod := corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: build.Namespace, + Name: buildPodName(build.Spec.Meta), + Labels: map[string]string{ + "camel.apache.org/build": build.Name, + }, + }, + } + + err := client.Delete(ctx, &pod) + if err != nil && k8serrors.IsNotFound(err) { + return nil + } + + return err +} + +// getBuilderPod -- +func getBuilderPod(ctx context.Context, client k8sclient.Reader, build *v1alpha1.Build) (*corev1.Pod, error) { + key := k8sclient.ObjectKey{Namespace: build.Namespace, Name: buildPodName(build.ObjectMeta)} + pod := corev1.Pod{} + + err := client.Get(ctx, key, &pod) + if err != nil && k8serrors.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, err + } + + return &pod, nil +} + +func buildPodName(object metav1.ObjectMeta) string { + return "camel-k-" + object.Name + "-builder" +}