Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WiP] adopt Compose-go to parse Compose files v2 or v3 #1440

Closed
wants to merge 1 commit into from
Closed

[WiP] adopt Compose-go to parse Compose files v2 or v3 #1440

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 3 additions & 8 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ go 1.13

replace github.com/Sirupsen/logrus => github.com/sirupsen/logrus v1.6.0

replace github.com/docker/libcompose => github.com/docker/libcompose v0.4.1-0.20190808084053-143e0f3f1ab9

replace github.com/docker/cli => github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible

replace github.com/xeipuuv/gojsonschema => github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5
Expand All @@ -17,29 +15,26 @@ replace github.com/containerd/containerd => github.com/containerd/containerd v1.
replace golang.org/x/sys => golang.org/x/sys v0.0.0-20201029080932-201ba4db2418

require (
github.com/compose-spec/compose-go v1.0.2
github.com/deckarep/golang-set v1.7.1
github.com/docker/cli v0.0.0-20190711175710-5b38d82aa076
github.com/docker/go-connections v0.4.0
github.com/docker/libcompose v0.4.0
github.com/fatih/structs v1.1.0
github.com/fsouza/go-dockerclient v1.6.5
github.com/google/go-cmp v0.4.0
github.com/google/go-cmp v0.5.5
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/imdario/mergo v0.3.10 // indirect
github.com/joho/godotenv v1.3.0
github.com/moby/sys/mount v0.1.1 // indirect
github.com/moby/term v0.0.0-20200915141129-7f0af18e79f2 // indirect
github.com/novln/docker-parser v1.0.0
github.com/openshift/api v0.0.0-20200803131051-87466835fcc0
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.6.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cast v1.3.1
github.com/spf13/cobra v1.0.0
github.com/spf13/viper v1.7.1
golang.org/x/tools v0.1.1
gopkg.in/yaml.v2 v2.3.0
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
gotest.tools/v3 v3.0.3 // indirect
k8s.io/api v0.19.0-rc.2
k8s.io/apimachinery v0.19.0-rc.2
)
83 changes: 56 additions & 27 deletions go.sum
View file
Open in desktop

Large diffs are not rendered by default.

109 changes: 54 additions & 55 deletions pkg/kobject/kobject.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@ import (
"path/filepath"
"time"

dockerCliTypes "github.com/docker/cli/cli/compose/types"
"github.com/docker/libcompose/yaml"
compose "github.com/compose-spec/compose-go/types"
deployapi "github.com/openshift/api/apps/v1"
"github.com/pkg/errors"
"github.com/spf13/cast"
Expand All @@ -37,7 +36,7 @@ type KomposeObject struct {
// as they can have different names. For example environment variables are called environment in compose but Env in bundle.
LoadedFrom string

Secrets map[string]dockerCliTypes.SecretConfig
Secrets compose.Secrets
}

// ConvertOptions holds all options that controls transformation process
Expand Down Expand Up @@ -96,63 +95,63 @@ type ServiceConfigGroup []ServiceConfig
type ServiceConfig struct {
Name string
ContainerName string
Image string `compose:"image"`
Environment []EnvVar `compose:"environment"`
EnvFile []string `compose:"env_file"`
Port []Ports `compose:"ports"`
Command []string `compose:"command"`
WorkingDir string `compose:""`
DomainName string `compose:"domainname"`
HostName string `compose:"hostname"`
Args []string `compose:"args"`
VolList []string `compose:"volumes"`
Network []string `compose:"network"`
Labels map[string]string `compose:"labels"`
Annotations map[string]string `compose:""`
CPUSet string `compose:"cpuset"`
CPUShares int64 `compose:"cpu_shares"`
CPUQuota int64 `compose:"cpu_quota"`
CPULimit int64 `compose:""`
CPUReservation int64 `compose:""`
CapAdd []string `compose:"cap_add"`
CapDrop []string `compose:"cap_drop"`
Expose []string `compose:"expose"`
ImagePullPolicy string `compose:"kompose.image-pull-policy"`
Pid string `compose:"pid"`
Privileged bool `compose:"privileged"`
Restart string `compose:"restart"`
User string `compose:"user"`
VolumesFrom []string `compose:"volumes_from"`
ServiceType string `compose:"kompose.service.type"`
NodePortPort int32 `compose:"kompose.service.nodeport.port"`
StopGracePeriod string `compose:"stop_grace_period"`
Build string `compose:"build"`
BuildArgs map[string]*string `compose:"build-args"`
ExposeService string `compose:"kompose.service.expose"`
ExposeServicePath string `compose:"kompose.service.expose.path"`
BuildLabels map[string]string `compose:"build-labels"`
ExposeServiceTLS string `compose:"kompose.service.expose.tls-secret"`
ImagePullSecret string `compose:"kompose.image-pull-secret"`
Stdin bool `compose:"stdin_open"`
Tty bool `compose:"tty"`
MemLimit yaml.MemStringorInt `compose:"mem_limit"`
MemReservation yaml.MemStringorInt `compose:""`
DeployMode string `compose:""`
Image string `compose:"image"`
Environment []EnvVar `compose:"environment"`
EnvFile []string `compose:"env_file"`
Port []Ports `compose:"ports"`
Command []string `compose:"command"`
WorkingDir string `compose:""`
DomainName string `compose:"domainname"`
HostName string `compose:"hostname"`
Args []string `compose:"args"`
VolList []string `compose:"volumes"`
Network []string `compose:"network"`
Labels map[string]string `compose:"labels"`
Annotations map[string]string `compose:""`
CPUSet string `compose:"cpuset"`
CPUShares int64 `compose:"cpu_shares"`
CPUQuota int64 `compose:"cpu_quota"`
CPULimit int64 `compose:""`
CPUReservation int64 `compose:""`
CapAdd []string `compose:"cap_add"`
CapDrop []string `compose:"cap_drop"`
Expose []string `compose:"expose"`
ImagePullPolicy string `compose:"kompose.image-pull-policy"`
Pid string `compose:"pid"`
Privileged bool `compose:"privileged"`
Restart string `compose:"restart"`
User string `compose:"user"`
VolumesFrom []string `compose:"volumes_from"`
ServiceType string `compose:"kompose.service.type"`
NodePortPort int32 `compose:"kompose.service.nodeport.port"`
StopGracePeriod string `compose:"stop_grace_period"`
Build string `compose:"build"`
BuildArgs map[string]*string `compose:"build-args"`
ExposeService string `compose:"kompose.service.expose"`
ExposeServicePath string `compose:"kompose.service.expose.path"`
BuildLabels map[string]string `compose:"build-labels"`
ExposeServiceTLS string `compose:"kompose.service.expose.tls-secret"`
ImagePullSecret string `compose:"kompose.image-pull-secret"`
Stdin bool `compose:"stdin_open"`
Tty bool `compose:"tty"`
MemLimit int64 `compose:"mem_limit"`
MemReservation int64 `compose:""`
DeployMode string `compose:""`
// DeployLabels mapping to kubernetes labels
DeployLabels map[string]string `compose:""`
DeployUpdateConfig dockerCliTypes.UpdateConfig `compose:""`
TmpFs []string `compose:"tmpfs"`
Dockerfile string `compose:"dockerfile"`
Replicas int `compose:"replicas"`
GroupAdd []int64 `compose:"group_add"`
Volumes []Volumes `compose:""`
Secrets []dockerCliTypes.ServiceSecretConfig
DeployLabels map[string]string `compose:""`
DeployUpdateConfig compose.UpdateConfig `compose:""`
TmpFs []string `compose:"tmpfs"`
Dockerfile string `compose:"dockerfile"`
Replicas int `compose:"replicas"`
GroupAdd []int64 `compose:"group_add"`
Volumes []Volumes `compose:""`
Secrets []compose.ServiceSecretConfig
HealthChecks HealthChecks `compose:""`
Placement Placement `compose:""`
//This is for long LONG SYNTAX link(https://docs.docker.com/compose/compose-file/#long-syntax)
Configs []dockerCliTypes.ServiceConfigObjConfig `compose:""`
Configs []compose.ServiceConfigObjConfig `compose:""`
//This is for SHORT SYNTAX link(https://docs.docker.com/compose/compose-file/#configs)
ConfigsMetaData map[string]dockerCliTypes.ConfigObjConfig `compose:""`
ConfigsMetaData map[string]compose.ConfigObjConfig `compose:""`

WithKomposeAnnotation bool `compose:""`
InGroup bool
Expand Down
179 changes: 15 additions & 164 deletions pkg/loader/compose/compose.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,8 @@ limitations under the License.
package compose

import (
"fmt"
"reflect"
"strings"

"gopkg.in/yaml.v2"

"github.com/docker/libcompose/project"
"github.com/fatih/structs"
"github.com/compose-spec/compose-go/cli"
"github.com/kubernetes/kompose/pkg/kobject"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)

Expand All @@ -37,168 +29,27 @@ var StdinData []byte
type Compose struct {
}

// checkUnsupportedKey checks if libcompose project contains
// keys that are not supported by this loader.
// list of all unsupported keys are stored in unsupportedKey variable
// returns list of unsupported YAML keys from docker-compose
func checkUnsupportedKey(composeProject *project.Project) []string {
// list of all unsupported keys for this loader
// this is map to make searching for keys easier
// to make sure that unsupported key is not going to be reported twice
// by keeping record if already saw this key in another service
var unsupportedKey = map[string]bool{
"CgroupParent": false,
"CPUSet": false,
"CPUShares": false,
"Devices": false,
"DependsOn": false,
"DNS": false,
"DNSSearch": false,
"EnvFile": false,
"ExternalLinks": false,
"ExtraHosts": false,
"Ipc": false,
"Logging": false,
"MacAddress": false,
"MemSwapLimit": false,
"NetworkMode": false,
"SecurityOpt": false,
"ShmSize": false,
"StopSignal": false,
"VolumeDriver": false,
"Uts": false,
"ReadOnly": false,
"Ulimits": false,
"Net": false,
"Sysctls": false,
//"Networks": false, // We shall be spporting network now. There are special checks for Network in checkUnsupportedKey function
"Links": false,
}

var keysFound []string

// Root level keys are not yet supported except Network
// Check to see if the default network is available and length is only equal to one.
if _, ok := composeProject.NetworkConfigs["default"]; ok && len(composeProject.NetworkConfigs) == 1 {
log.Debug("Default network found")
}

// Root level volumes are not yet supported
if len(composeProject.VolumeConfigs) > 0 {
keysFound = append(keysFound, "root level volumes")
}

for _, serviceConfig := range composeProject.ServiceConfigs.All() {
// this reflection is used in check for empty arrays
val := reflect.ValueOf(serviceConfig).Elem()
s := structs.New(serviceConfig)

for _, f := range s.Fields() {
// Check if given key is among unsupported keys, and skip it if we already saw this key
if alreadySaw, ok := unsupportedKey[f.Name()]; ok && !alreadySaw {
if f.IsExported() && !f.IsZero() {
// IsZero returns false for empty array/slice ([])
// this check if field is Slice, and then it checks its size
if field := val.FieldByName(f.Name()); field.Kind() == reflect.Slice {
if field.Len() == 0 {
// array is empty it doesn't matter if it is in unsupportedKey or not
continue
}
}
//get yaml tag name instead of variable name
yamlTagName := strings.Split(f.Tag("yaml"), ",")[0]
if f.Name() == "Networks" {
// networks always contains one default element, even it isn't declared in compose v2.
if len(serviceConfig.Networks.Networks) == 1 && serviceConfig.Networks.Networks[0].Name == "default" {
// this is empty Network definition, skip it
continue
}
}

if linksArray := val.FieldByName(f.Name()); f.Name() == "Links" && linksArray.Kind() == reflect.Slice {
//Links has "SERVICE:ALIAS" style, we don't support SERVICE != ALIAS
findUnsupportedLinksFlag := false
for i := 0; i < linksArray.Len(); i++ {
if tmpLink := linksArray.Index(i); tmpLink.Kind() == reflect.String {
tmpLinkStr := tmpLink.String()
tmpLinkStrSplit := strings.Split(tmpLinkStr, ":")
if len(tmpLinkStrSplit) == 2 && tmpLinkStrSplit[0] != tmpLinkStrSplit[1] {
findUnsupportedLinksFlag = true
break
}
}
}
if !findUnsupportedLinksFlag {
continue
}
}

keysFound = append(keysFound, yamlTagName)
unsupportedKey[f.Name()] = true
}
}
}
}
return keysFound
}

// LoadFile loads a compose file into KomposeObject
func (c *Compose) LoadFile(files []string) (kobject.KomposeObject, error) {
// Load the json / yaml file in order to get the version value
var version string

for _, file := range files {
composeVersion, err := getVersionFromFile(file)
if err != nil {
return kobject.KomposeObject{}, errors.Wrap(err, "Unable to load yaml/json file for version parsing")
}

// Check that the previous file loaded matches.
if len(files) > 0 && version != "" && version != composeVersion {
return kobject.KomposeObject{}, errors.New("All Docker Compose files must be of the same version")
}
version = composeVersion
}

log.Debugf("Docker Compose version: %s", version)

// Convert based on version
switch version {
// Use libcompose for 1 or 2
// If blank, it's assumed it's 1 or 2
case "", "1", "1.0", "2", "2.0", "2.1", "2.2":
komposeObject, err := parseV1V2(files)
if err != nil {
return kobject.KomposeObject{}, err
}
return komposeObject, nil
// Use docker/cli for 3
case "3", "3.0", "3.1", "3.2", "3.3", "3.4", "3.5", "3.6", "3.7", "3.8":
komposeObject, err := parseV3(files)
if err != nil {
return kobject.KomposeObject{}, err
}
return komposeObject, nil
default:
return kobject.KomposeObject{}, fmt.Errorf("version %s of Docker Compose is not supported. Please use version 1, 2 or 3", version)
}
}

func getVersionFromFile(file string) (string, error) {
type ComposeVersion struct {
Version string `json:"version"` // This affects YAML as well
options, err := cli.NewProjectOptions(files,
cli.WithDotEnv,
cli.WithOsEnv,
cli.WithConfigFileEnv,
cli.WithDefaultConfigPath)
if err != nil {
return kobject.KomposeObject{}, err
}
var version ComposeVersion
loadedFile, err := ReadFile(file)

project, err := cli.ProjectFromOptions(options)
if err != nil {
return "", err
return kobject.KomposeObject{}, err
}

err = yaml.Unmarshal(loadedFile, &version)
if err != nil {
return "", err
noSupKeys := checkUnsupportedKeyForV3(project)
for _, keyName := range noSupKeys {
log.Warningf("Unsupported %s key - ignoring", keyName)
}

return version.Version, nil
// Finally, we convert the object from docker/cli's ServiceConfig to our appropriate one
return dockerComposeToKomposeMapping(project)
}
Loading