diff --git a/pkg/component/component.go b/pkg/component/component.go index 9cb5bca32f6..12edfa9327a 100644 --- a/pkg/component/component.go +++ b/pkg/component/component.go @@ -675,7 +675,7 @@ func PushLocal(client *occlient.Client, componentName string, applicationName st compInfo := common.ComponentInfo{ PodName: pod.Name, } - err = sync.CopyFile(client, path, compInfo, targetPath, files, globExps, map[string][]byte{}) + err = sync.CopyFile(client, path, compInfo, targetPath, files, globExps) if err != nil { s.End(false) return errors.Wrap(err, "unable push files to pod") diff --git a/pkg/devfile/adapters/kubernetes/component/adapter.go b/pkg/devfile/adapters/kubernetes/component/adapter.go index 32c672eee46..df6e37d9be6 100644 --- a/pkg/devfile/adapters/kubernetes/component/adapter.go +++ b/pkg/devfile/adapters/kubernetes/component/adapter.go @@ -1,8 +1,10 @@ package component import ( + "bufio" "bytes" "fmt" + "io" "os" "path/filepath" "reflect" @@ -21,8 +23,6 @@ import ( "github.com/pkg/errors" "k8s.io/klog" - "github.com/openshift/odo/pkg/occlient" - "github.com/openshift/odo/pkg/component" "github.com/openshift/odo/pkg/config" "github.com/openshift/odo/pkg/devfile/adapters/common" @@ -34,6 +34,7 @@ import ( "github.com/openshift/odo/pkg/exec" "github.com/openshift/odo/pkg/kclient" "github.com/openshift/odo/pkg/log" + "github.com/openshift/odo/pkg/occlient" odoutil "github.com/openshift/odo/pkg/odo/util" "github.com/openshift/odo/pkg/sync" "github.com/openshift/odo/pkg/url" @@ -56,165 +57,87 @@ type Adapter struct { devfileRunCmd string } -func (a Adapter) generateBuildContainer(containerName, imageTag string) corev1.Container { - buildImage := "quay.io/buildah/stable:latest" - - // TODO(Optional): Init container before the buildah bud to copy over the files. - //command := []string{"buildah"} - //commandArgs := []string{"bud"} - command := []string{"tail"} - commandArgs := []string{"-f", "/dev/null"} - - // TODO: Edit dockerfile env value if mounting it sometwhere else - envVars := []corev1.EnvVar{ - {Name: "Tag", Value: imageTag}, - } - - isPrivileged := true - resourceReqs := corev1.ResourceRequirements{} +const dockerfilePath string = "Dockerfile" - container := kclient.GenerateContainer(containerName, buildImage, isPrivileged, command, commandArgs, envVars, resourceReqs, nil) +func (a Adapter) runBuildConfig(client *occlient.Client, parameters common.BuildParameters) (err error) { + buildName := a.ComponentName - container.VolumeMounts = []corev1.VolumeMount{ - {Name: "varlibcontainers", MountPath: "/var/lib/containers"}, - {Name: kclient.OdoSourceVolume, MountPath: kclient.OdoSourceVolumeMount}, + commonObjectMeta := metav1.ObjectMeta{ + Name: buildName, } - return *container -} - -func (a Adapter) createBuildDeployment(labels map[string]string, container corev1.Container) (err error) { - - objectMeta := kclient.CreateObjectMeta(a.ComponentName, a.Client.Namespace, labels, nil) - podTemplateSpec := kclient.GeneratePodTemplateSpec(objectMeta, []corev1.Container{container}) - - // TODO: For openshift, need to specify a service account that allows priviledged containers - saEnv := os.Getenv("BUILD_SERVICE_ACCOUNT") - if saEnv != "" { - podTemplateSpec.Spec.ServiceAccountName = saEnv - } - - libContainersVolume := corev1.Volume{ - Name: "varlibcontainers", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - } - - podTemplateSpec.Spec.Volumes = append(podTemplateSpec.Spec.Volumes, libContainersVolume) - - deploymentSpec := kclient.GenerateDeploymentSpec(*podTemplateSpec) - klog.V(3).Infof("Creating deployment %v", deploymentSpec.Template.GetName()) - - _, err = a.Client.CreateDeployment(*deploymentSpec) + _, err = client.CreateDockerBuildConfigWithBinaryInput(commonObjectMeta, dockerfilePath, parameters.Tag, []corev1.EnvVar{}) if err != nil { return err } - klog.V(3).Infof("Successfully created component %v", deploymentSpec.Template.GetName()) - return nil -} - -func (a Adapter) executeBuildAndPush(syncFolder string, imageTag string, compInfo common.ComponentInfo) (err error) { - // Running buildah bud and buildah push - buildahBud := "buildah bud -f ./Dockerfile -t $Tag ." - command := []string{adaptersCommon.ShellExecutable, "-c", "cd " + syncFolder + " && " + buildahBud} + defer func() { + // This will delete both the BuildConfig and any builds using that BuildConfig + derr := client.DeleteBuildConfig(commonObjectMeta) + if err == nil { + err = derr + } + }() - // TODO: Add spinner - err = exec.ExecuteCommand(&a.Client, compInfo, command, false) + syncAdapter := sync.New(a.AdapterContext, &a.Client) + reader, err := syncAdapter.SyncFilesBuild(parameters, dockerfilePath) if err != nil { - return errors.Wrapf(err, "failed to build image for component with name: %s", a.ComponentName) - } - - values := strings.Split(imageTag, "/") - tag := imageTag - buildahPush := "buildah push " - - // Need to change this IF to be more robust - if len(values) == 3 && strings.Contains(values[0], "openshift") { - // This needs a valid service account: e.g builder for openshift - // --creds flag arg has the format username:password - // we want to use serviceaccount:token - buildahPush += "--creds " - saEnv := os.Getenv("BUILD_SERVICE_ACCOUNT") - if saEnv != "" { - buildahPush += saEnv - } else { - buildahBud += "dummy-username" - } - buildahPush += ":$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) " + return err } - //TODO: handle dockerhub case and creds!! - buildahPush += "--tls-verify=false " + tag + " docker://" + tag - command = []string{adaptersCommon.ShellExecutable, "-c", buildahPush} - - //TODO: Add Spinner - err = exec.ExecuteCommand(&a.Client, compInfo, command, false) + bc, err := client.RunBuildConfigWithBinaryInput(buildName, reader) if err != nil { - return errors.Wrapf(err, "failed to push build image to the registry for component with name: %s", a.ComponentName) + return err } - return nil -} + reader, writer := io.Pipe() + s := log.Spinner("Waiting for build to finish") -// Build image for devfile project -func (a Adapter) Build(parameters common.BuildParameters) (err error) { - containerName := a.ComponentName + "-container" - buildContainer := a.generateBuildContainer(containerName, parameters.Tag) - labels := map[string]string{ - "component": a.ComponentName, - } + var cmdOutput string + // This Go routine will automatically pipe the output from WaitForBuildToFinish to + // our logger. + go func() { + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + line := scanner.Text() - err = a.createBuildDeployment(labels, buildContainer) - if err != nil { - return errors.Wrap(err, "error while creating buildah deployment") - } + if log.IsDebug() { + _, err := fmt.Fprintln(os.Stdout, line) + if err != nil { + log.Errorf("Unable to print to stdout: %v", err) + } + } - // Delete deployment - defer func() { - derr := a.Delete(labels) - if err == nil { - err = errors.Wrapf(derr, "failed to delete build step for component with name: %s", a.ComponentName) + cmdOutput += fmt.Sprintln(line) } - }() - _, err = a.Client.WaitForDeploymentRollout(a.ComponentName) - if err != nil { - return errors.Wrap(err, "error while waiting for deployment rollout") - } - - // Wait for Pod to be in running state otherwise we can't sync data or exec commands to it. - pod, err := a.waitAndGetComponentPod(false) - if err != nil { - return errors.Wrapf(err, "unable to get pod for component %s", a.ComponentName) - } - - // Need to wait for container to start - time.Sleep(10 * time.Second) - - // Sync files to volume - log.Infof("\nSyncing to component %s", a.ComponentName) - // Get a sync adapter. Check if project files have changed and sync accordingly - syncAdapter := sync.New(a.AdapterContext, &a.Client) - compInfo := common.ComponentInfo{ - ContainerName: containerName, - PodName: pod.GetName(), + if err := client.WaitForBuildToFinish(bc.Name, writer); err != nil { + s.End(false) + return errors.Wrapf(err, "unable to build image using BuildConfig %s, error: %s", buildName, cmdOutput) } - syncFolder, err := syncAdapter.SyncFilesBuild(parameters, compInfo) + s.End(true) + return +} +// Build image for devfile project +func (a Adapter) Build(parameters common.BuildParameters) (err error) { + client, err := occlient.New() if err != nil { - return errors.Wrapf(err, "failed to sync to component with name %s", a.ComponentName) + return err } - err = a.executeBuildAndPush(syncFolder, parameters.Tag, compInfo) + isBuildConfigSupported, err := client.IsBuildConfigSupported() if err != nil { return err } - return + if isBuildConfigSupported { + return a.runBuildConfig(client, parameters) + } + + return errors.New("unable to build image, only Openshift BuildConfig build is supported") } func determinePort(envSpecificInfo envinfo.EnvSpecificInfo) string { @@ -302,7 +225,6 @@ func (a Adapter) waitForManifestDeployCompletion(applicationName string, gvr sch // Build image for devfile project func (a Adapter) Deploy(parameters common.DeployParameters) (err error) { - namespace := a.Client.Namespace applicationName := a.ComponentName + "-deploy" deploymentManifest := &unstructured.Unstructured{} diff --git a/pkg/occlient/occlient.go b/pkg/occlient/occlient.go index 62605768390..3e91ae806d6 100644 --- a/pkg/occlient/occlient.go +++ b/pkg/occlient/occlient.go @@ -396,6 +396,31 @@ func (c *Client) GetPortsFromBuilderImage(componentType string) ([]string, error return portList, nil } +// RunBuildConfigWithBinary +func (c *Client) RunBuildConfigWithBinaryInput(name string, r io.Reader) (*buildv1.Build, error) { + // TODO: investigate this issue + // Error: no kind is registered for the type v1.BinaryBuildRequestOptions in scheme "k8s.io/client-go/kubernetes/scheme/register.go:69" + // options := &buildv1.BinaryBuildRequestOptions{ + // ObjectMeta: metav1.ObjectMeta{ + // Name: name, + // Namespace: c.Namespace, + // }, + // } + result := &buildv1.Build{} + err := c.buildClient.RESTClient(). + Post(). + Namespace(c.Namespace). + Resource("buildconfigs"). + Name(name). + SubResource("instantiatebinary"). + Body(r). + //VersionedParams(options, scheme.ParameterCodec). + Do(). + Into(result) + + return result, err +} + // RunLogout logs out the current user from cluster func (c *Client) RunLogout(stdout io.Writer) error { output, err := c.userClient.Users().Get("~", metav1.GetOptions{}) @@ -3112,6 +3137,23 @@ func (c *Client) GetPVCFromName(pvcName string) (*corev1.PersistentVolumeClaim, return c.kubeClient.CoreV1().PersistentVolumeClaims(c.Namespace).Get(pvcName, metav1.GetOptions{}) } +// CreateDockerBuildConfigWithBinaryAndDockerfile creates a BuildConfig which accepts +// as input a binary which contains a dockerfile at a specific location. It will build +// the source with Dockerfile, and push the image using tag. +// envVars is the array containing the environment variables +func (c *Client) CreateDockerBuildConfigWithBinaryInput(commonObjectMeta metav1.ObjectMeta, dockerfilePath string, outputImageTag string, envVars []corev1.EnvVar) (bc buildv1.BuildConfig, err error) { + bc = generateDockerBuildConfigWithBinaryInput(commonObjectMeta, dockerfilePath, outputImageTag) + + if len(envVars) > 0 { + bc.Spec.Strategy.SourceStrategy.Env = envVars + } + _, err = c.buildClient.BuildConfigs(c.Namespace).Create(&bc) + if err != nil { + return bc, errors.Wrapf(err, "unable to create BuildConfig for %s", commonObjectMeta.Name) + } + return bc, err +} + // CreateBuildConfig creates a buildConfig using the builderImage as well as gitURL. // envVars is the array containing the environment variables func (c *Client) CreateBuildConfig(commonObjectMeta metav1.ObjectMeta, builderImage string, gitURL string, gitRef string, envVars []corev1.EnvVar) (buildv1.BuildConfig, error) { @@ -3298,6 +3340,12 @@ func isSubDir(baseDir, otherDir string) bool { return matches } +// IsBuildConfigSupported checks if buildconfig resource type is present on the cluster +func (c *Client) IsBuildConfigSupported() (bool, error) { + + return c.isResourceSupported("build.openshift.io", "v1", "buildconfigs") +} + // IsRouteSupported checks if route resource type is present on the cluster func (c *Client) IsRouteSupported() (bool, error) { diff --git a/pkg/occlient/templates.go b/pkg/occlient/templates.go index 2fa6e4c6225..60ecf58083e 100644 --- a/pkg/occlient/templates.go +++ b/pkg/occlient/templates.go @@ -315,6 +315,38 @@ func generateGitDeploymentConfig(commonObjectMeta metav1.ObjectMeta, image strin return dc } +// generateDockerBuildConfigWithBinaryInput creates a BuildConfig which accepts a Binary (usualy archive) +// with a Dockerfile at the path specified. It will run a build using docker, and push the resulting image +// to the tag specified. +func generateDockerBuildConfigWithBinaryInput(commonObjectMeta metav1.ObjectMeta, dockerfilePath, outputImageTag string) buildv1.BuildConfig { + buildSource := buildv1.BuildSource{ + Binary: &buildv1.BinaryBuildSource{}, + Type: buildv1.BuildSourceBinary, + } + + return buildv1.BuildConfig{ + ObjectMeta: commonObjectMeta, + Spec: buildv1.BuildConfigSpec{ + CommonSpec: buildv1.CommonSpec{ + Output: buildv1.BuildOutput{ + To: &corev1.ObjectReference{ + Kind: "DockerImage", + Name: outputImageTag, + }, + // OPTIONAL: PushSecrets and ImageLabels + }, + Source: buildSource, + Strategy: buildv1.BuildStrategy{ + Type: buildv1.DockerBuildStrategyType, + DockerStrategy: &buildv1.DockerBuildStrategy{ + DockerfilePath: dockerfilePath, + }, + }, + }, + }, + } +} + // generateBuildConfig creates a BuildConfig for Git URL's being passed into Odo func generateBuildConfig(commonObjectMeta metav1.ObjectMeta, gitURL, gitRef, imageName, imageNamespace string) buildv1.BuildConfig { diff --git a/pkg/odo/cli/component/component.go b/pkg/odo/cli/component/component.go index d46ee492975..4ab41a3ec0f 100644 --- a/pkg/odo/cli/component/component.go +++ b/pkg/odo/cli/component/component.go @@ -9,6 +9,7 @@ import ( "github.com/openshift/odo/pkg/log" "github.com/openshift/odo/pkg/occlient" "github.com/openshift/odo/pkg/odo/util/completion" + "github.com/openshift/odo/pkg/odo/util/experimental" "github.com/openshift/odo/pkg/url" "github.com/pkg/errors" @@ -70,7 +71,11 @@ func NewCmdComponent(name, fullName string) *cobra.Command { // add flags from 'get' to component command componentCmd.Flags().AddFlagSet(componentGetCmd.Flags()) - componentCmd.AddCommand(componentGetCmd, createCmd, deleteCmd, describeCmd, deployCmd, linkCmd, unlinkCmd, listCmd, logCmd, pushCmd, updateCmd, watchCmd) + componentCmd.AddCommand(componentGetCmd, createCmd, deleteCmd, describeCmd, linkCmd, unlinkCmd, listCmd, logCmd, pushCmd, updateCmd, watchCmd) + + if experimental.IsExperimentalModeEnabled() { + componentCmd.AddCommand(deployCmd) + } // Add a defined annotation in order to appear in the help menu componentCmd.Annotations = map[string]string{"command": "main"} diff --git a/pkg/sync/adapter.go b/pkg/sync/adapter.go index 6442ffc35e3..9c20892c1a9 100644 --- a/pkg/sync/adapter.go +++ b/pkg/sync/adapter.go @@ -2,6 +2,7 @@ package sync import ( "fmt" + "io" "os" "path/filepath" "strings" @@ -32,7 +33,7 @@ type Adapter struct { } // SyncFilesBuild sync the local files to build container volume -func (a Adapter) SyncFilesBuild(buildParameters common.BuildParameters, compInfo common.ComponentInfo) (syncFolder string, err error) { +func (a Adapter) SyncFilesBuild(buildParameters common.BuildParameters, dockerfilePath string) (reader io.Reader, err error) { // If we want to ignore any files absIgnoreRules := []string{} @@ -41,7 +42,7 @@ func (a Adapter) SyncFilesBuild(buildParameters common.BuildParameters, compInfo } var s *log.Status - syncFolder = "/projects" + syncFolder := "/" s = log.Spinner("Checking files for deploy") // run the indexer and find the project source files @@ -49,30 +50,15 @@ func (a Adapter) SyncFilesBuild(buildParameters common.BuildParameters, compInfo if len(files) > 0 { klog.V(4).Infof("Copying files %s to pod", strings.Join(files, " ")) dockerfile := map[string][]byte{ - "Dockerfile": buildParameters.DockerfileBytes, - } - err = CopyFile(a.Client, buildParameters.Path, compInfo, syncFolder, files, absIgnoreRules, dockerfile) - if err != nil { - s.End(false) - return syncFolder, errors.Wrap(err, "unable push files to pod") + dockerfilePath: buildParameters.DockerfileBytes, } + reader, err = GetTarReader(buildParameters.Path, syncFolder, files, absIgnoreRules, dockerfile) + s.End(true) + return reader, err } - - // TODO: We may want to be using push local for consistency and because it has a delete path incase the volume remains in the cluster. - // We could also change the SyncFiles function directly with a build setting. - // err = a.pushLocal(pushParameters.Path, - // changedFiles, - // deletedFiles, - // isForcePush, - // globExps, - // compInfo, - // ) - // if err != nil { - // return false, errors.Wrapf(err, "failed to sync to component with name %s", a.ComponentName) - // } + klog.V(4).Infof("No files to sync") s.End(true) - - return syncFolder, nil + return reader, nil } // SyncFiles does a couple of things: @@ -220,7 +206,7 @@ func (a Adapter) pushLocal(path string, files []string, delFiles []string, isFor if isForcePush || len(files) > 0 { klog.V(4).Infof("Copying files %s to pod", strings.Join(files, " ")) - err = CopyFile(a.Client, path, compInfo, syncFolder, files, globExps, map[string][]byte{}) + err = CopyFile(a.Client, path, compInfo, syncFolder, files, globExps) if err != nil { s.End(false) return errors.Wrap(err, "unable push files to pod") diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index c48c4a0c801..bf35c67075e 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -19,17 +19,11 @@ type SyncClient interface { ExtractProjectToComponent(common.ComponentInfo, string, io.Reader) error } -// CopyFile copies localPath directory or list of files in copyFiles list to the directory in running Pod. -// copyFiles is list of changed files captured during `odo watch` as well as binary file path -// During copying binary components, localPath represent base directory path to binary and copyFiles contains path of binary -// During copying local source components, localPath represent base directory path whereas copyFiles is empty -// During `odo watch`, localPath represent base directory path whereas copyFiles contains list of changed Files -func CopyFile(client SyncClient, localPath string, compInfo common.ComponentInfo, targetPath string, copyFiles []string, globExps []string, copyBytes map[string][]byte) error { - +// GetTarReader creates a tar file from files and return a reader to it +func GetTarReader(localPath string, targetPath string, copyFiles []string, globExps []string, copyBytes map[string][]byte) (reader io.Reader, err error) { // Destination is set to "ToSlash" as all containers being ran within OpenShift / S2I are all // Linux based and thus: "\opt\app-root\src" would not work correctly. dest := filepath.ToSlash(filepath.Join(targetPath, filepath.Base(localPath))) - targetPath = filepath.ToSlash(targetPath) klog.V(4).Infof("CopyFile arguments: localPath %s, dest %s, targetPath %s, copyFiles %s, globalExps %s", localPath, dest, targetPath, copyFiles, globExps) reader, writer := io.Pipe() @@ -67,7 +61,38 @@ func CopyFile(client SyncClient, localPath string, compInfo common.ComponentInfo } tarWriter.Close() + }() + + return reader, err +} + +// CopyFile copies localPath directory or list of files in copyFiles list to the directory in running Pod. +// copyFiles is list of changed files captured during `odo watch` as well as binary file path +// During copying binary components, localPath represent base directory path to binary and copyFiles contains path of binary +// During copying local source components, localPath represent base directory path whereas copyFiles is empty +// During `odo watch`, localPath represent base directory path whereas copyFiles contains list of changed Files +func CopyFile(client SyncClient, localPath string, compInfo common.ComponentInfo, targetPath string, copyFiles []string, globExps []string) error { + + // Destination is set to "ToSlash" as all containers being ran within OpenShift / S2I are all + // Linux based and thus: "\opt\app-root\src" would not work correctly. + dest := filepath.ToSlash(filepath.Join(targetPath, filepath.Base(localPath))) + targetPath = filepath.ToSlash(targetPath) + + klog.V(4).Infof("CopyFile arguments: localPath %s, dest %s, targetPath %s, copyFiles %s, globalExps %s", localPath, dest, targetPath, copyFiles, globExps) + reader, writer := io.Pipe() + // inspired from https://github.com/kubernetes/kubernetes/blob/master/pkg/kubectl/cmd/cp.go#L235 + go func() { + defer writer.Close() + tarWriter := tar.NewWriter(writer) + + err := makeTar(localPath, dest, copyFiles, globExps, tarWriter) + if err != nil { + log.Errorf("Error while creating tar: %#v", err) + os.Exit(1) + } + + tarWriter.Close() }() err := client.ExtractProjectToComponent(compInfo, targetPath, reader)