Skip to content

Commit

Permalink
adopt Compose-go to parse Compose files v2 or v3
Browse files Browse the repository at this point in the history 
Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>
  • Loading branch information
@ndeloof
ndeloof committed Oct 4, 2021
1 parent 72ea6a3 commit 052d97e
Show file tree
Hide file tree
Showing 8 changed files with 163 additions and 921 deletions.
11 changes: 3 additions & 8 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ go 1.13

replace github.com/Sirupsen/logrus => github.com/sirupsen/logrus v1.6.0

replace github.com/docker/libcompose => github.com/docker/libcompose v0.4.1-0.20190808084053-143e0f3f1ab9

replace github.com/docker/cli => github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible

replace github.com/xeipuuv/gojsonschema => github.com/xeipuuv/gojsonschema v1.2.1-0.20201027075954-b076d39a02e5
Expand All @@ -17,29 +15,26 @@ replace github.com/containerd/containerd => github.com/containerd/containerd v1.
replace golang.org/x/sys => golang.org/x/sys v0.0.0-20201029080932-201ba4db2418

require (
github.com/compose-spec/compose-go v1.0.2
github.com/deckarep/golang-set v1.7.1
github.com/docker/cli v0.0.0-20190711175710-5b38d82aa076
github.com/docker/go-connections v0.4.0
github.com/docker/libcompose v0.4.0
github.com/fatih/structs v1.1.0
github.com/fsouza/go-dockerclient v1.6.5
github.com/google/go-cmp v0.4.0
github.com/google/go-cmp v0.5.5
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/imdario/mergo v0.3.10 // indirect
github.com/joho/godotenv v1.3.0
github.com/moby/sys/mount v0.1.1 // indirect
github.com/moby/term v0.0.0-20200915141129-7f0af18e79f2 // indirect
github.com/novln/docker-parser v1.0.0
github.com/openshift/api v0.0.0-20200803131051-87466835fcc0
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.6.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cast v1.3.1
github.com/spf13/cobra v1.0.0
github.com/spf13/viper v1.7.1
golang.org/x/tools v0.1.1
gopkg.in/yaml.v2 v2.3.0
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
gotest.tools/v3 v3.0.3 // indirect
k8s.io/api v0.19.0-rc.2
k8s.io/apimachinery v0.19.0-rc.2
)
83 changes: 56 additions & 27 deletions go.sum
View file

Large diffs are not rendered by default.

109 changes: 54 additions & 55 deletions pkg/kobject/kobject.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@ import (
"path/filepath"
"time"

dockerCliTypes "github.com/docker/cli/cli/compose/types"
"github.com/docker/libcompose/yaml"
compose "github.com/compose-spec/compose-go/types"
deployapi "github.com/openshift/api/apps/v1"
"github.com/pkg/errors"
"github.com/spf13/cast"
Expand All @@ -37,7 +36,7 @@ type KomposeObject struct {
// as they can have different names. For example environment variables are called environment in compose but Env in bundle.
LoadedFrom string

Secrets map[string]dockerCliTypes.SecretConfig
Secrets compose.Secrets
}

// ConvertOptions holds all options that controls transformation process
Expand Down Expand Up @@ -96,63 +95,63 @@ type ServiceConfigGroup []ServiceConfig
type ServiceConfig struct {
Name string
ContainerName string
Image string `compose:"image"`
Environment []EnvVar `compose:"environment"`
EnvFile []string `compose:"env_file"`
Port []Ports `compose:"ports"`
Command []string `compose:"command"`
WorkingDir string `compose:""`
DomainName string `compose:"domainname"`
HostName string `compose:"hostname"`
Args []string `compose:"args"`
VolList []string `compose:"volumes"`
Network []string `compose:"network"`
Labels map[string]string `compose:"labels"`
Annotations map[string]string `compose:""`
CPUSet string `compose:"cpuset"`
CPUShares int64 `compose:"cpu_shares"`
CPUQuota int64 `compose:"cpu_quota"`
CPULimit int64 `compose:""`
CPUReservation int64 `compose:""`
CapAdd []string `compose:"cap_add"`
CapDrop []string `compose:"cap_drop"`
Expose []string `compose:"expose"`
ImagePullPolicy string `compose:"kompose.image-pull-policy"`
Pid string `compose:"pid"`
Privileged bool `compose:"privileged"`
Restart string `compose:"restart"`
User string `compose:"user"`
VolumesFrom []string `compose:"volumes_from"`
ServiceType string `compose:"kompose.service.type"`
NodePortPort int32 `compose:"kompose.service.nodeport.port"`
StopGracePeriod string `compose:"stop_grace_period"`
Build string `compose:"build"`
BuildArgs map[string]*string `compose:"build-args"`
ExposeService string `compose:"kompose.service.expose"`
ExposeServicePath string `compose:"kompose.service.expose.path"`
BuildLabels map[string]string `compose:"build-labels"`
ExposeServiceTLS string `compose:"kompose.service.expose.tls-secret"`
ImagePullSecret string `compose:"kompose.image-pull-secret"`
Stdin bool `compose:"stdin_open"`
Tty bool `compose:"tty"`
MemLimit yaml.MemStringorInt `compose:"mem_limit"`
MemReservation yaml.MemStringorInt `compose:""`
DeployMode string `compose:""`
Image string `compose:"image"`
Environment []EnvVar `compose:"environment"`
EnvFile []string `compose:"env_file"`
Port []Ports `compose:"ports"`
Command []string `compose:"command"`
WorkingDir string `compose:""`
DomainName string `compose:"domainname"`
HostName string `compose:"hostname"`
Args []string `compose:"args"`
VolList []string `compose:"volumes"`
Network []string `compose:"network"`
Labels map[string]string `compose:"labels"`
Annotations map[string]string `compose:""`
CPUSet string `compose:"cpuset"`
CPUShares int64 `compose:"cpu_shares"`
CPUQuota int64 `compose:"cpu_quota"`
CPULimit int64 `compose:""`
CPUReservation int64 `compose:""`
CapAdd []string `compose:"cap_add"`
CapDrop []string `compose:"cap_drop"`
Expose []string `compose:"expose"`
ImagePullPolicy string `compose:"kompose.image-pull-policy"`
Pid string `compose:"pid"`
Privileged bool `compose:"privileged"`
Restart string `compose:"restart"`
User string `compose:"user"`
VolumesFrom []string `compose:"volumes_from"`
ServiceType string `compose:"kompose.service.type"`
NodePortPort int32 `compose:"kompose.service.nodeport.port"`
StopGracePeriod string `compose:"stop_grace_period"`
Build string `compose:"build"`
BuildArgs map[string]*string `compose:"build-args"`
ExposeService string `compose:"kompose.service.expose"`
ExposeServicePath string `compose:"kompose.service.expose.path"`
BuildLabels map[string]string `compose:"build-labels"`
ExposeServiceTLS string `compose:"kompose.service.expose.tls-secret"`
ImagePullSecret string `compose:"kompose.image-pull-secret"`
Stdin bool `compose:"stdin_open"`
Tty bool `compose:"tty"`
MemLimit int64 `compose:"mem_limit"`
MemReservation int64 `compose:""`
DeployMode string `compose:""`
// DeployLabels mapping to kubernetes labels
DeployLabels map[string]string `compose:""`
DeployUpdateConfig dockerCliTypes.UpdateConfig `compose:""`
TmpFs []string `compose:"tmpfs"`
Dockerfile string `compose:"dockerfile"`
Replicas int `compose:"replicas"`
GroupAdd []int64 `compose:"group_add"`
Volumes []Volumes `compose:""`
Secrets []dockerCliTypes.ServiceSecretConfig
DeployLabels map[string]string `compose:""`
DeployUpdateConfig compose.UpdateConfig `compose:""`
TmpFs []string `compose:"tmpfs"`
Dockerfile string `compose:"dockerfile"`
Replicas int `compose:"replicas"`
GroupAdd []int64 `compose:"group_add"`
Volumes []Volumes `compose:""`
Secrets []compose.ServiceSecretConfig
HealthChecks HealthChecks `compose:""`
Placement Placement `compose:""`
//This is for long LONG SYNTAX link(https://docs.docker.com/compose/compose-file/#long-syntax)
Configs []dockerCliTypes.ServiceConfigObjConfig `compose:""`
Configs []compose.ServiceConfigObjConfig `compose:""`
//This is for SHORT SYNTAX link(https://docs.docker.com/compose/compose-file/#configs)
ConfigsMetaData map[string]dockerCliTypes.ConfigObjConfig `compose:""`
ConfigsMetaData map[string]compose.ConfigObjConfig `compose:""`

WithKomposeAnnotation bool `compose:""`
InGroup bool
Expand Down
179 changes: 15 additions & 164 deletions pkg/loader/compose/compose.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,8 @@ limitations under the License.
package compose

import (
"fmt"
"reflect"
"strings"

"gopkg.in/yaml.v2"

"github.com/docker/libcompose/project"
"github.com/fatih/structs"
"github.com/compose-spec/compose-go/cli"
"github.com/kubernetes/kompose/pkg/kobject"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)

Expand All @@ -37,168 +29,27 @@ var StdinData []byte
type Compose struct {
}

// checkUnsupportedKey checks if libcompose project contains
// keys that are not supported by this loader.
// list of all unsupported keys are stored in unsupportedKey variable
// returns list of unsupported YAML keys from docker-compose
func checkUnsupportedKey(composeProject *project.Project) []string {
// list of all unsupported keys for this loader
// this is map to make searching for keys easier
// to make sure that unsupported key is not going to be reported twice
// by keeping record if already saw this key in another service
var unsupportedKey = map[string]bool{
"CgroupParent": false,
"CPUSet": false,
"CPUShares": false,
"Devices": false,
"DependsOn": false,
"DNS": false,
"DNSSearch": false,
"EnvFile": false,
"ExternalLinks": false,
"ExtraHosts": false,
"Ipc": false,
"Logging": false,
"MacAddress": false,
"MemSwapLimit": false,
"NetworkMode": false,
"SecurityOpt": false,
"ShmSize": false,
"StopSignal": false,
"VolumeDriver": false,
"Uts": false,
"ReadOnly": false,
"Ulimits": false,
"Net": false,
"Sysctls": false,
//"Networks": false, // We shall be spporting network now. There are special checks for Network in checkUnsupportedKey function
"Links": false,
}

var keysFound []string

// Root level keys are not yet supported except Network
// Check to see if the default network is available and length is only equal to one.
if _, ok := composeProject.NetworkConfigs["default"]; ok && len(composeProject.NetworkConfigs) == 1 {
log.Debug("Default network found")
}

// Root level volumes are not yet supported
if len(composeProject.VolumeConfigs) > 0 {
keysFound = append(keysFound, "root level volumes")
}

for _, serviceConfig := range composeProject.ServiceConfigs.All() {
// this reflection is used in check for empty arrays
val := reflect.ValueOf(serviceConfig).Elem()
s := structs.New(serviceConfig)

for _, f := range s.Fields() {
// Check if given key is among unsupported keys, and skip it if we already saw this key
if alreadySaw, ok := unsupportedKey[f.Name()]; ok && !alreadySaw {
if f.IsExported() && !f.IsZero() {
// IsZero returns false for empty array/slice ([])
// this check if field is Slice, and then it checks its size
if field := val.FieldByName(f.Name()); field.Kind() == reflect.Slice {
if field.Len() == 0 {
// array is empty it doesn't matter if it is in unsupportedKey or not
continue
}
}
//get yaml tag name instead of variable name
yamlTagName := strings.Split(f.Tag("yaml"), ",")[0]
if f.Name() == "Networks" {
// networks always contains one default element, even it isn't declared in compose v2.
if len(serviceConfig.Networks.Networks) == 1 && serviceConfig.Networks.Networks[0].Name == "default" {
// this is empty Network definition, skip it
continue
}
}

if linksArray := val.FieldByName(f.Name()); f.Name() == "Links" && linksArray.Kind() == reflect.Slice {
//Links has "SERVICE:ALIAS" style, we don't support SERVICE != ALIAS
findUnsupportedLinksFlag := false
for i := 0; i < linksArray.Len(); i++ {
if tmpLink := linksArray.Index(i); tmpLink.Kind() == reflect.String {
tmpLinkStr := tmpLink.String()
tmpLinkStrSplit := strings.Split(tmpLinkStr, ":")
if len(tmpLinkStrSplit) == 2 && tmpLinkStrSplit[0] != tmpLinkStrSplit[1] {
findUnsupportedLinksFlag = true
break
}
}
}
if !findUnsupportedLinksFlag {
continue
}
}

keysFound = append(keysFound, yamlTagName)
unsupportedKey[f.Name()] = true
}
}
}
}
return keysFound
}

// LoadFile loads a compose file into KomposeObject
func (c *Compose) LoadFile(files []string) (kobject.KomposeObject, error) {
// Load the json / yaml file in order to get the version value
var version string

for _, file := range files {
composeVersion, err := getVersionFromFile(file)
if err != nil {
return kobject.KomposeObject{}, errors.Wrap(err, "Unable to load yaml/json file for version parsing")
}

// Check that the previous file loaded matches.
if len(files) > 0 && version != "" && version != composeVersion {
return kobject.KomposeObject{}, errors.New("All Docker Compose files must be of the same version")
}
version = composeVersion
}

log.Debugf("Docker Compose version: %s", version)

// Convert based on version
switch version {
// Use libcompose for 1 or 2
// If blank, it's assumed it's 1 or 2
case "", "1", "1.0", "2", "2.0", "2.1", "2.2":
komposeObject, err := parseV1V2(files)
if err != nil {
return kobject.KomposeObject{}, err
}
return komposeObject, nil
// Use docker/cli for 3
case "3", "3.0", "3.1", "3.2", "3.3", "3.4", "3.5", "3.6", "3.7", "3.8":
komposeObject, err := parseV3(files)
if err != nil {
return kobject.KomposeObject{}, err
}
return komposeObject, nil
default:
return kobject.KomposeObject{}, fmt.Errorf("version %s of Docker Compose is not supported. Please use version 1, 2 or 3", version)
}
}

func getVersionFromFile(file string) (string, error) {
type ComposeVersion struct {
Version string `json:"version"` // This affects YAML as well
options, err := cli.NewProjectOptions(files,
cli.WithDotEnv,
cli.WithOsEnv,
cli.WithConfigFileEnv,
cli.WithDefaultConfigPath)
if err != nil {
return kobject.KomposeObject{}, err
}
var version ComposeVersion
loadedFile, err := ReadFile(file)

project, err := cli.ProjectFromOptions(options)
if err != nil {
return "", err
return kobject.KomposeObject{}, err
}

err = yaml.Unmarshal(loadedFile, &version)
if err != nil {
return "", err
noSupKeys := checkUnsupportedKeyForV3(project)
for _, keyName := range noSupKeys {
log.Warningf("Unsupported %s key - ignoring", keyName)
}

return version.Version, nil
// Finally, we convert the object from docker/cli's ServiceConfig to our appropriate one
return dockerComposeToKomposeMapping(project)
}
Loading

0 comments on commit 052d97e

Please sign in to comment.