diff --git a/docs/book/tutorials/deploying_cpi_and_csi_with_multi_dc_vc_aka_zones.md b/docs/book/tutorials/deploying_cpi_and_csi_with_multi_dc_vc_aka_zones.md index 82f4e9f04..574a73128 100644 --- a/docs/book/tutorials/deploying_cpi_and_csi_with_multi_dc_vc_aka_zones.md +++ b/docs/book/tutorials/deploying_cpi_and_csi_with_multi_dc_vc_aka_zones.md @@ -67,35 +67,42 @@ Steps that will be covered in order to setup zones for the vSphere CPI, vSphere > ***Note:*** The CSI and CPI drivers have their own vsphere.conf files. The following modifications need to be made in both configurations. -The zones implementation depends on 2 sets of vSphere tags to be used on objects, such as datacenters or clusters. The first is a `region` tag and the second is a `zone` tag. vSphere tags are very simply put key/value pairs that can be assigned to objects and instead of using fixed keys to denote a `region` or a `zone`, we give the end-user the ability to come up with their own keys for a `region` and `zone` in the form of vSphere Tag Catagory. It just allows for a level of indirection in case you already have regions and zones setup in your configuration. Once a key/label or vSphere Tag Category is selected for each, create a `[Labels]` section in the `vsphere.conf` then assign tag names for both `region` and `zone`. +The zones implementation depends on 2 sets of vSphere tags to be used on objects, such as datacenters or clusters. The first is a `region` tag and the second is a `zone` tag. vSphere tags are very simply put key/value pairs that can be assigned to objects and instead of using fixed keys to denote a `region` or a `zone`, we give the end-user the ability to come up with their own keys for a `region` and `zone` in the form of vSphere Tag Catagory. It just allows for a level of indirection in case you already have regions and zones setup in your configuration. Once a key/label or vSphere Tag Category is selected for each, create a `labels:` section in the `vsphere.conf` then assign tag names for both `region` and `zone`. In the example `vsphere.conf` below, `k8s-region` and `k8s-zone` was selected: ```bash -[Global] -# properties in this section will be used for all specified vCenters unless overridden in VirtualCenter section. - -user = "vCenter username for cloud provider" -password = "password" -port = "443" #Optional -insecure-flag = "1" #set to 1 if the vCenter uses a self-signed cert -datacenters = "list of datacenters where Kubernetes node VMs are present" - -[VirtualCenter "1.2.3.4"] -# Override specific properties for this Virtual Center. - user = "vCenter username for cloud provider" - password = "password" - # port, insecure-flag, datacenters will be used from Global section. - -[VirtualCenter "10.0.0.1"] -# Override specific properties for this Virtual Center. - port = "448" - insecure-flag = "0" - # user, password, datacenters will be used from Global section. - -[Labels] -region = k8s-region -zone = k8s-zone +# Global properties in this section will be used for all specified vCenters unless overriden in VirtualCenter section. +global: + user: YourVCenterUser + password: YourVCenterPass + port: "443" + # set insecureFlag to true if the vCenter uses a self-signed cert + insecureFlag: true + # settings for using k8s secret + secretName: cpi-secret + secretNamespace: kube-system + +# VirtualCenter section +vcenter: + tenant1: + user: YourVCenterUser + password: YourVCenterPass + server: 10.0.0.1 + datacenters: + - mydc1 + tenant2: + server: 127.0.0.1 + port: 448 + insecureFlag: false + datacenters: + - myotherdc1 + - myotherdc2 + +# labels for regions and zones +labels: + region: k8s-region + zone: k8s-zone ``` ### 2. Creating Zones in your vSphere Environment via Tags diff --git a/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md b/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md index 0ebab3041..f2f307a33 100644 --- a/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md +++ b/docs/book/tutorials/kubernetes-on-vsphere-with-kubeadm.md @@ -477,34 +477,51 @@ This cloud-config configmap file, passed to the CPI on initialization, contains ```bash # tee /etc/kubernetes/vsphere.conf >/dev/null <`, the vCenter IP address in the keys of `stringData`, and the `username` and `password` for each key. diff --git a/go.mod b/go.mod index 671c81508..a82fb1cd0 100644 --- a/go.mod +++ b/go.mod @@ -5,18 +5,17 @@ go 1.12 require ( github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/evanphx/json-patch v4.5.0+incompatible // indirect github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/golang/protobuf v1.3.2 github.com/google/btree v1.0.0 // indirect github.com/google/uuid v1.1.1 github.com/imdario/mergo v0.3.7 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect - github.com/pkg/errors v0.8.0 + github.com/pkg/errors v0.9.0 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 - github.com/spf13/cobra v0.0.3 - github.com/spf13/pflag v1.0.3 - github.com/vmware/govmomi v0.21.0 + github.com/spf13/cobra v0.0.5 + github.com/spf13/pflag v1.0.5 + github.com/vmware/govmomi v0.22.1 github.com/vmware/vsphere-automation-sdk-go/lib v0.1.1 github.com/vmware/vsphere-automation-sdk-go/runtime v0.1.1 github.com/vmware/vsphere-automation-sdk-go/services/nsxt v0.1.1 @@ -28,9 +27,10 @@ require ( gopkg.in/gcfg.v1 v1.2.3 gopkg.in/square/go-jose.v2 v2.3.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.2.7 honnef.co/go/tools v0.0.1-2020.1.3 // indirect k8s.io/api v0.0.0 - k8s.io/apimachinery v0.0.0 + k8s.io/apimachinery v0.17.0 k8s.io/client-go v0.0.0 k8s.io/cloud-provider v0.0.0 k8s.io/component-base v0.0.0 @@ -38,6 +38,7 @@ require ( k8s.io/kube-openapi v0.0.0-20190401085232-94e1e7b7574c // indirect k8s.io/kubernetes v1.15.0 k8s.io/sample-controller v0.0.0-20190731144349-6f8905ae4ee5 + sigs.k8s.io/kind v0.7.0 // indirect ) replace ( diff --git a/go.sum b/go.sum index 31e266da4..e15c3e278 100644 --- a/go.sum +++ b/go.sum @@ -23,7 +23,10 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/Rican7/retry v0.1.0/go.mod h1:FgOROf8P5bebcC1DS0PdOQiqGUridaZvikzUmkFW6gg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053 h1:H/GMMKYPkEIC3DF/JWQz8Pdd+Feifov2EIgGfNpeogI= +github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053/go.mod h1:xW8sBma2LE3QxFSzCnH9qe6gAE2yO9GvQaWwX89HxbE= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1:LWMyo4iOLWXHGdBki7NIht1kHru/0wM179h+d3g8ATM= github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -50,17 +53,21 @@ github.com/containerd/typeurl v0.0.0-20190228175220-2a93cfde8c20/go.mod h1:Cm3kw github.com/containernetworking/cni v0.6.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d00KFj3A= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v0.0.0-20180117170138-065b426bd416/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.0.0-20180108230905-e214231b295a h1:WqY2Kv7eI1jeoU3pC05YYK/kK4tdXyLzzaBzCR51r9M= github.com/coreos/go-semver v0.0.0-20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea h1:n2Ltr3SrfQlf/9nOna1DoGKxLx3qTSI8Ttl6Xrqp6mw= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/rkt v1.30.0/go.mod h1:O634mlH6U7qk87poQifK6M2rsFNt+FyUTWNMnP1hF1U= github.com/cpuguy83/go-md2man v1.0.4/go.mod h1:N6JayAiVKtlHSnuTCeuLSQVs75hb8q+dYQLjr7cDsKY= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cyphar/filepath-securejoin v0.0.0-20170720062807-ae69057f2299/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= github.com/d2g/dhcp4client v0.0.0-20170829104524-6e570ed0a266/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= @@ -180,6 +187,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v0.0.0-20160711231752-d8c773c4cba1/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/heketi/heketi v0.0.0-20181109135656-558b29266ce0/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o= github.com/heketi/rest v0.0.0-20180404230133-aa6a65207413/go.mod h1:BeS3M108VzVlmAue3lv2WcGuPAX94/KN63MUURzbYSI= github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4= @@ -220,9 +228,12 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA= github.com/magiconair/properties v0.0.0-20160816085511-61b492c03cf4/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marstr/guid v0.0.0-20170427235115-8bdf7d1a087c/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-shellwords v0.0.0-20180605041737-f8471b0a71de/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -231,6 +242,7 @@ github.com/mholt/caddy v0.0.0-20180213163048-2de495001514/go.mod h1:Wb1PlT4DAYSq github.com/miekg/dns v0.0.0-20160614162101-5d001d020961/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mindprince/gonvml v0.0.0-20171110221305-fee913ce8fb2/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= github.com/mistifyio/go-zfs v0.0.0-20151009155749-1b4ae6fb4e77/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -262,9 +274,13 @@ github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= +github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.0 h1:J8lpUdobwIeCI7OiSxHqEwJUKvJwicL5+3v1oe2Yb4k= +github.com/pkg/errors v0.9.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v0.0.0-20160930220758-4d0e916071f6 h1:V8AT/I4KmIDRfObq0yBUvbD4DeaYmQY9GhC5sKl24Mo= github.com/pkg/sftp v0.0.0-20160930220758-4d0e916071f6/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -291,6 +307,7 @@ github.com/robfig/cron v0.0.0-20170309132418-df38d32658d8/go.mod h1:JGuDeoQd7Z6y github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v0.0.0-20151117072312-300106c228d5/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/seccomp/libseccomp-golang v0.0.0-20150813023252-1b506fc7c24e/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/shurcooL/sanitized_anchor_name v0.0.0-20151028001915-10ef21a441db/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -303,15 +320,23 @@ github.com/soheilhy/cmux v0.1.3 h1:09wy7WZk4AqO03yH85Ex1X+Uo3vDsil3Fa9AgF8Emss= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v0.0.0-20160816080757-b28a7effac97 h1:Gv1HykSEG+RKWWWkM69nPrJKhE/EM2oFb1nBWogHNv8= github.com/spf13/afero v0.0.0-20160816080757-b28a7effac97/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v0.0.0-20160730092037-e31f36ffc91a/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.0-20180319062004-c439c4fa0937/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v0.0.0-20160311093646-33c24e77fb80/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v0.0.0-20160820190039-7fb2782df3d8/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/storageos/go-api v0.0.0-20180912212459-343b3eff91fc/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= @@ -319,12 +344,15 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/syndtr/gocapability v0.0.0-20160928074757-e7cb7fa329f4/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vmware/govmomi v0.20.1/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/vmware/govmomi v0.21.0 h1:jc8uMuxpcV2xMAA/cnEDlnsIjvqcMra5Y8onh/U3VuY= github.com/vmware/govmomi v0.21.0/go.mod h1:zbnFoBQ9GIjs2RVETy8CNEpb+L+Lwkjs3XZUL0B3/m0= +github.com/vmware/govmomi v0.22.1 h1:ZIEYmBdAS2i+s7RctapqdHfbeGiUcL8LRN05uS4TfPc= +github.com/vmware/govmomi v0.22.1/go.mod h1:Y+Wq4lst78L85Ge/F8+ORXIWiKYqaro1vhAulACy9Lc= github.com/vmware/photon-controller-go-sdk v0.0.0-20170310013346-4a435daef6cc/go.mod h1:e6humHha1ekIwTCm+A5Qed5mG8V4JL+ChHcUOJ+L/8U= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/vmware/vsphere-automation-sdk-go/lib v0.1.1 h1:PmDaeuToX1QKKe9VWRJztAp2/IyjbbGZp6fEiff4Dr8= @@ -337,6 +365,7 @@ github.com/xanzy/go-cloudstack v0.0.0-20160728180336-1e2cbf647e57/go.mod h1:s3eL github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18 h1:MPPkRncZLN9Kh4MEFmbnK4h3BD7AUmskWv2+EeZJCCs= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569 h1:nSQar3Y0E3VQF/VdZ8PTAilaXpER+d7ypdABCrpwMdg= go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df h1:shvkWr0NAZkg4nPuE3XrKP0VuBPijjk3TfX6Y6acFNg= @@ -345,8 +374,10 @@ go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15 h1:Z2sc4+v0JHV6Mn4kX1f2a5nruN go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -386,11 +417,15 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181004145325-8469e314837c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313 h1:pczuHS43Cp2ktBEEmLwScxgjWsBSzdaQiKzUyf3DTTc= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1 h1:gZpLHxUX5BdYLA08Lj4YCJNN/jk7KtquiArPoeX0WvA= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -455,9 +490,14 @@ gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+p gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 h1:XZx7nhd5GMaZpmDaEHFVafUZC7ya0fuo7cSJ3UCKYmM= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= @@ -509,6 +549,8 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +sigs.k8s.io/kind v0.7.0 h1:7y7a8EYtGHM+auHmsvzuK5o84SrxPYGidlvfql7j/k4= +sigs.k8s.io/kind v0.7.0/go.mod h1:An/AbWHT6pA/Lm0Og8j3ukGhfJP3RiVN/IBU6Lo3zl8= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2 h1:9r5DY45ef9LtcA6BnkhW8MPV7OKAfbf2AUwUhq3LeRk= sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= diff --git a/manifests/controller-manager/vsphere-cloud-controller-manager-ds.yaml b/manifests/controller-manager/vsphere-cloud-controller-manager-ds.yaml index ca4794579..769b1c643 100644 --- a/manifests/controller-manager/vsphere-cloud-controller-manager-ds.yaml +++ b/manifests/controller-manager/vsphere-cloud-controller-manager-ds.yaml @@ -8,10 +8,14 @@ metadata: apiVersion: apps/v1 kind: DaemonSet metadata: - name: vsphere-cloud-controller-manager - namespace: kube-system + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" labels: + component: cloud-controller-manager + tier: control-plane k8s-app: vsphere-cloud-controller-manager + name: vsphere-cloud-controller-manager + namespace: kube-system spec: selector: matchLabels: @@ -26,7 +30,7 @@ spec: nodeSelector: node-role.kubernetes.io/master: "" securityContext: - runAsUser: 0 + runAsUser: 1001 tolerations: - key: node.cloudprovider.kubernetes.io/uninitialized value: "true" diff --git a/manifests/controller-manager/vsphere-cloud-controller-manager-pod.yaml b/manifests/controller-manager/vsphere-cloud-controller-manager-pod.yaml index f4c1549a6..c3f2b7056 100644 --- a/manifests/controller-manager/vsphere-cloud-controller-manager-pod.yaml +++ b/manifests/controller-manager/vsphere-cloud-controller-manager-pod.yaml @@ -13,6 +13,7 @@ metadata: labels: component: cloud-controller-manager tier: control-plane + k8s-app: vsphere-cloud-controller-manager name: vsphere-cloud-controller-manager namespace: kube-system spec: @@ -25,7 +26,7 @@ spec: - --cloud-provider=vsphere volumeMounts: - mountPath: /etc/cloud - name: cloud-config-volume + name: vsphere-config-volume readOnly: true resources: requests: @@ -42,7 +43,7 @@ spec: runAsUser: 1001 serviceAccountName: cloud-controller-manager volumes: - - name: cloud-config-volume + - name: vsphere-config-volume configMap: name: cloud-config --- diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index f20601006..100fb48ae 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -21,8 +21,11 @@ import ( "encoding/base64" "encoding/pem" "fmt" + "io/ioutil" "os" + "k8s.io/klog" + "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/ssoadmin" "github.com/vmware/govmomi/ssoadmin/types" @@ -46,7 +49,14 @@ func ParseConfig(configFile string) (*config.Config, error) { if err != nil { return nil, fmt.Errorf("Can not open config file %s, %v", configFile, err) } - cfg, err := config.ReadConfig(f) + + byConfig, err := ioutil.ReadAll(f) + if err != nil { + klog.Errorf("ReadAll failed: %s", err) + return nil, err + } + + cfg, err := config.ReadConfig(byConfig) if err != nil { return nil, err } diff --git a/pkg/cloudprovider/vsphere/cloud.go b/pkg/cloudprovider/vsphere/cloud.go index 6afd7f0cc..79a8af797 100644 --- a/pkg/cloudprovider/vsphere/cloud.go +++ b/pkg/cloudprovider/vsphere/cloud.go @@ -17,18 +17,21 @@ limitations under the License. package vsphere import ( - "fmt" "io" + "io/ioutil" "os" "runtime" v1 "k8s.io/api/core/v1" - cloudprovider "k8s.io/cloud-provider" "k8s.io/klog" + cloudprovider "k8s.io/cloud-provider" + "github.com/vmware/vsphere-automation-sdk-go/runtime/log" + ccfg "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/config" "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/loadbalancer" + lcfg "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/loadbalancer/config" "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/server" cm "k8s.io/cloud-provider-vsphere/pkg/common/connectionmanager" k8s "k8s.io/cloud-provider-vsphere/pkg/common/kubernetes" @@ -44,17 +47,28 @@ const ( func init() { cloudprovider.RegisterCloudProvider(ProviderName, func(config io.Reader) (cloudprovider.Interface, error) { - cpiConfig, err := ReadCPIConfig(config) + byConfig, err := ioutil.ReadAll(config) + if err != nil { + klog.Errorf("ReadAll failed: %s", err) + return nil, err + } + + cfg, err := ccfg.ReadCPIConfig(byConfig) if err != nil { return nil, err } - return newVSphere(cpiConfig, true) + lbcfg, err := lcfg.ReadLBConfig(byConfig) + if err != nil { + lbcfg = nil //Error reading LBConfig, explicitly set to nil + } + + return newVSphere(cfg, lbcfg, true) }) } // Creates new Controller node interface and returns -func newVSphere(cfg *CPIConfig, finalize ...bool) (*VSphere, error) { - vs, err := buildVSphereFromConfig(cfg) +func newVSphere(cfg *ccfg.CPIConfig, lbcfg *lcfg.LBConfig, finalize ...bool) (*VSphere, error) { + vs, err := buildVSphereFromConfig(cfg, lbcfg) if err != nil { return nil, err } @@ -161,31 +175,30 @@ func (vs *VSphere) HasClusterID() bool { } // Initializes vSphere from vSphere CloudProvider Configuration -func buildVSphereFromConfig(cfg *CPIConfig) (*VSphere, error) { +func buildVSphereFromConfig(cfg *ccfg.CPIConfig, lbcfg *lcfg.LBConfig) (*VSphere, error) { nm := newNodeManager(cfg, nil) - lb, err := loadbalancer.NewLBProvider(&cfg.LBConfig) + + lb, err := loadbalancer.NewLBProvider(lbcfg) if err != nil { return nil, err } - if _, ok := os.LookupEnv("ENABLE_ALPHA_NSXT_LB"); !ok { - if lb != nil { - klog.Infof("To enable NSX-T load balancer support you need to set the env variable ENABLE_ALPHA_NSXT_LB") - lb = nil - } - } else { + if _, ok := os.LookupEnv("ENABLE_ALPHA_NSXT_LB"); ok { if lb == nil { - return nil, fmt.Errorf("To enable NSX-T load balancer support you need to configure section LoadBalancer") + klog.Warning("To enable NSX-T load balancer support you need to configure section LoadBalancer") + } else { + klog.Infof("NSX-T load balancer support enabled. This feature is alpha, use in production at your own risk.") + // redirect vapi logging from the NSX-T GO SDK to klog + log.SetLogger(NewKlogBridge()) } - } - if lb == nil { - klog.Infof("NSX-T load balancer support disabled") } else { - klog.Infof("NSX-T load balancer support enabled. This feature is alpha, use in production at your own risk.") - // redirect vapi logging from the NSX-T GO SDK to klog - log.SetLogger(NewKlogBridge()) + // explicitly nil the LB interface if ENABLE_ALPHA_NSXT_LB is not set even if the LBConfig is valid + // ENABLE_ALPHA_NSXT_LB must be explicitly enabled + lb = nil } + vs := VSphere{ cfg: cfg, + cfgLB: lbcfg, nodeManager: nm, loadbalancer: lb, instances: newInstances(nm), diff --git a/pkg/cloudprovider/vsphere/config.go b/pkg/cloudprovider/vsphere/config/config.go similarity index 66% rename from pkg/cloudprovider/vsphere/config.go rename to pkg/cloudprovider/vsphere/config/config.go index 3a2b4fa27..b6c7bbc25 100644 --- a/pkg/cloudprovider/vsphere/config.go +++ b/pkg/cloudprovider/vsphere/config/config.go @@ -14,14 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package vsphere +package config import ( "fmt" - "io" "os" - "gopkg.in/gcfg.v1" + "k8s.io/klog" ) // FromCPIEnv initializes the provided configuration object with values @@ -50,27 +49,42 @@ func (cfg *CPIConfig) FromCPIEnv() error { return nil } +/* + TODO: + When the INI based cloud-config is deprecated, the references to the + INI based code (ie the call to ReadConfigINI) below should be deleted. +*/ + // ReadCPIConfig parses vSphere cloud config file and stores it into CPIConfig. // Environment variables are also checked -func ReadCPIConfig(config io.Reader) (*CPIConfig, error) { - if config == nil { - return nil, fmt.Errorf("no vSphere cloud provider config file given") +func ReadCPIConfig(byConfig []byte) (*CPIConfig, error) { + if len(byConfig) == 0 { + err := fmt.Errorf("no vSphere cloud provider config file given") + klog.Error("config is nil") + return nil, err } - cfg := &CPIConfig{} + cfg, err := ReadCPIConfigYAML(byConfig) + if err != nil { + klog.Warningf("ReadCPIConfigYAML failed: %s", err) - if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, config)); err != nil { - return nil, err + cfg, err = ReadCPIConfigINI(byConfig) + if err != nil { + klog.Errorf("ReadConfigINI failed: %s", err) + return nil, err + } + + klog.Info("ReadConfig INI succeeded. CPI INI-based cloud-config is deprecated and will be removed in 2.0. Please use YAML based cloud-config.") + } else { + klog.Info("ReadConfig YAML succeeded") } // Env Vars should override config file entries if present if err := cfg.FromCPIEnv(); err != nil { + klog.Errorf("FromEnv failed: %s", err) return nil, err } - if err := cfg.LBConfig.CompleteAndValidate(); err != nil { - return nil, err - } - + klog.Info("Config initialized") return cfg, nil } diff --git a/pkg/cloudprovider/vsphere/config/config_ini_legacy.go b/pkg/cloudprovider/vsphere/config/config_ini_legacy.go new file mode 100644 index 000000000..218b2beec --- /dev/null +++ b/pkg/cloudprovider/vsphere/config/config_ini_legacy.go @@ -0,0 +1,72 @@ +/* +Copyright 2019 New The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "fmt" + + ini "gopkg.in/gcfg.v1" + + vcfg "k8s.io/cloud-provider-vsphere/pkg/common/config" +) + +/* + TODO: + When the INI based cloud-config is deprecated. This file should be deleted. +*/ + +// CreateConfig generates a common Config object based on what other structs and funcs +// are already dependent upon in other packages. +func (cci *CPIConfigINI) CreateConfig() *CPIConfig { + cfg := &CPIConfig{ + *cci.CommonConfigINI.CreateConfig(), + Nodes{ + InternalNetworkSubnetCIDR: cci.Nodes.InternalNetworkSubnetCIDR, + ExternalNetworkSubnetCIDR: cci.Nodes.ExternalNetworkSubnetCIDR, + InternalVMNetworkName: cci.Nodes.InternalVMNetworkName, + ExternalVMNetworkName: cci.Nodes.ExternalVMNetworkName, + }, + } + + return cfg +} + +// ReadCPIConfigINI parses vSphere cloud config file and stores it into CPIConfigYAML. +func ReadCPIConfigINI(byConfig []byte) (*CPIConfig, error) { + if len(byConfig) == 0 { + return nil, fmt.Errorf("Invalid INI file") + } + + strConfig := string(byConfig[:]) + + // Must grab the entire config then overwrite it... + cfgOLD := &CPIConfigINI{} + + if err := ini.FatalOnly(ini.ReadStringInto(cfgOLD, strConfig)); err != nil { + return nil, err + } + + // with this so that we can call the validate function within ReadRawConfigINI + vCFG, err := vcfg.ReadRawConfigINI(byConfig) + if err != nil { + return nil, err + } + + cfg := &CPIConfigINI{*vCFG, cfgOLD.Nodes} + + return cfg.CreateConfig(), nil +} diff --git a/pkg/cloudprovider/vsphere/config_test.go b/pkg/cloudprovider/vsphere/config/config_ini_legacy_test.go similarity index 78% rename from pkg/cloudprovider/vsphere/config_test.go rename to pkg/cloudprovider/vsphere/config/config_ini_legacy_test.go index 67c10adb7..841477f77 100644 --- a/pkg/cloudprovider/vsphere/config_test.go +++ b/pkg/cloudprovider/vsphere/config/config_ini_legacy_test.go @@ -11,14 +11,18 @@ See the License for the specific language governing permissions and limitations under the License. */ -package vsphere +package config import ( - "strings" "testing" ) -const subnetCidrConfig = ` +/* + TODO: + When the INI based cloud-config is deprecated. This file should be deleted. +*/ + +const subnetCidrINIConfig = ` [Global] server = 0.0.0.0 port = 443 @@ -33,7 +37,7 @@ internal-network-subnet-cidr = "192.0.2.0/24" external-network-subnet-cidr = "198.51.100.0/24" ` -const networkNameConfig = ` +const networkNameINIConfig = ` [Global] server = 0.0.0.0 port = 443 @@ -48,33 +52,36 @@ internal-vm-network-name = "Internal K8s Traffic" external-vm-network-name = "External/Outbound Traffic" ` -func TestReadConfigSubnetCidr(t *testing.T) { - _, err := ReadCPIConfig(nil) +func TestReadINIConfigSubnetCidr(t *testing.T) { + _, err := ReadCPIConfigINI(nil) if err == nil { t.Errorf("Should fail when no config is provided: %s", err) } - cfg, err := ReadCPIConfig(strings.NewReader(subnetCidrConfig)) + cfg, err := ReadCPIConfigINI([]byte(subnetCidrINIConfig)) if err != nil { t.Fatalf("Should succeed when a valid config is provided: %s", err) } + if cfg.Global.VCenterIP != "0.0.0.0" { + t.Errorf("incorrect global vcServerIP: %s", cfg.Global.VCenterIP) + } + if cfg.Nodes.InternalNetworkSubnetCIDR != "192.0.2.0/24" { t.Errorf("incorrect internal network subnet cidr: %s", cfg.Nodes.InternalNetworkSubnetCIDR) } - if cfg.Nodes.ExternalNetworkSubnetCIDR != "198.51.100.0/24" { t.Errorf("incorrect external network subnet cidr: %s", cfg.Nodes.ExternalNetworkSubnetCIDR) } } -func TestReadConfigNetworkName(t *testing.T) { - _, err := ReadCPIConfig(nil) +func TestReadINIConfigNetworkName(t *testing.T) { + _, err := ReadCPIConfigINI(nil) if err == nil { t.Errorf("Should fail when no config is provided: %s", err) } - cfg, err := ReadCPIConfig(strings.NewReader(networkNameConfig)) + cfg, err := ReadCPIConfigINI([]byte(networkNameINIConfig)) if err != nil { t.Fatalf("Should succeed when a valid config is provided: %s", err) } diff --git a/pkg/cloudprovider/vsphere/config/config_yaml.go b/pkg/cloudprovider/vsphere/config/config_yaml.go new file mode 100644 index 000000000..d53f02e95 --- /dev/null +++ b/pkg/cloudprovider/vsphere/config/config_yaml.go @@ -0,0 +1,71 @@ +/* +Copyright 2020 New The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "fmt" + + yaml "gopkg.in/yaml.v2" + + vcfg "k8s.io/cloud-provider-vsphere/pkg/common/config" +) + +/* + TODO: + When the INI based cloud-config is deprecated, this file should be renamed + from config_yaml.go to config.go +*/ + +// CreateConfig generates a common Config object based on what other structs and funcs +// are already dependent upon in other packages. +func (ccy *CPIConfigYAML) CreateConfig() *CPIConfig { + cfg := &CPIConfig{ + *ccy.CommonConfigYAML.CreateConfig(), + Nodes{ + InternalNetworkSubnetCIDR: ccy.Nodes.InternalNetworkSubnetCIDR, + ExternalNetworkSubnetCIDR: ccy.Nodes.ExternalNetworkSubnetCIDR, + InternalVMNetworkName: ccy.Nodes.InternalVMNetworkName, + ExternalVMNetworkName: ccy.Nodes.ExternalVMNetworkName, + }, + } + + return cfg +} + +// ReadCPIConfigYAML parses vSphere cloud config file and stores it into CPIConfigYAML. +func ReadCPIConfigYAML(byConfig []byte) (*CPIConfig, error) { + if len(byConfig) == 0 { + return nil, fmt.Errorf("Invalid YAML file") + } + + // Must grab the entire config then overwrite it... + cfgOLD := &CPIConfigYAML{} + + if err := yaml.Unmarshal(byConfig, cfgOLD); err != nil { + return nil, err + } + + // with this so that we can call the validate function within ReadRawConfigINI + vCFG, err := vcfg.ReadRawConfigYAML(byConfig) + if err != nil { + return nil, err + } + + cfg := &CPIConfigYAML{*vCFG, cfgOLD.Nodes} + + return cfg.CreateConfig(), nil +} diff --git a/pkg/cloudprovider/vsphere/config/config_yaml_test.go b/pkg/cloudprovider/vsphere/config/config_yaml_test.go new file mode 100644 index 000000000..ec17e98eb --- /dev/null +++ b/pkg/cloudprovider/vsphere/config/config_yaml_test.go @@ -0,0 +1,101 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "testing" +) + +/* + TODO: + When the INI based cloud-config is deprecated. This file should be deleted. +*/ + +const subnetCidrYAMLConfig = ` +global: + server: 0.0.0.0 + port: 443 + user: user + password: password + insecureFlag: true + datacenters: + - us-west + caFile: /some/path/to/a/ca.pem + +nodes: + internalNetworkSubnetCidr: 192.0.2.0/24 + externalNetworkSubnetCidr: 198.51.100.0/24 +` + +const networkNameYAMLConfig = ` +global: + server: 0.0.0.0 + port: 443 + user: user + password: password + insecureFlag: true + datacenters: + - us-west + caFile: /some/path/to/a/ca.pem + +nodes: + internalVmNetworkName: Internal K8s Traffic + externalVmNetworkName: External/Outbound Traffic +` + +func TestReadYAMLConfigSubnetCidr(t *testing.T) { + _, err := ReadCPIConfigYAML(nil) + if err == nil { + t.Errorf("Should fail when no config is provided: %s", err) + } + + cfg, err := ReadCPIConfigYAML([]byte(subnetCidrYAMLConfig)) + if err != nil { + t.Fatalf("Should succeed when a valid config is provided: %s", err) + } + + if cfg.Global.VCenterIP != "0.0.0.0" { + t.Errorf("incorrect global vcServerIP: %s", cfg.Global.VCenterIP) + } + + if cfg.Nodes.InternalNetworkSubnetCIDR != "192.0.2.0/24" { + t.Errorf("incorrect internal network subnet cidr: %s", cfg.Nodes.InternalNetworkSubnetCIDR) + } + if cfg.Nodes.ExternalNetworkSubnetCIDR != "198.51.100.0/24" { + t.Errorf("incorrect external network subnet cidr: %s", cfg.Nodes.ExternalNetworkSubnetCIDR) + } +} + +func TestReadYAMLConfigNetworkName(t *testing.T) { + _, err := ReadCPIConfigYAML(nil) + if err == nil { + t.Errorf("Should fail when no config is provided: %s", err) + } + + cfg, err := ReadCPIConfigYAML([]byte(networkNameYAMLConfig)) + if err != nil { + t.Fatalf("Should succeed when a valid config is provided: %s", err) + } + + if cfg.Nodes.InternalVMNetworkName != "Internal K8s Traffic" { + t.Errorf("incorrect internal vm network name: %s", cfg.Nodes.InternalVMNetworkName) + } + + if cfg.Nodes.ExternalVMNetworkName != "External/Outbound Traffic" { + t.Errorf("incorrect internal vm network name: %s", cfg.Nodes.ExternalVMNetworkName) + } +} diff --git a/pkg/cloudprovider/vsphere/config/types_common.go b/pkg/cloudprovider/vsphere/config/types_common.go new file mode 100644 index 000000000..412fd553b --- /dev/null +++ b/pkg/cloudprovider/vsphere/config/types_common.go @@ -0,0 +1,47 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + vcfg "k8s.io/cloud-provider-vsphere/pkg/common/config" +) + +/* + TODO: + When the INI based cloud-config is deprecated. This file should be deleted and + the structs in types_yaml.go will be renamed to replace the ones in this file. +*/ + +// Nodes captures internal/external networks +type Nodes struct { + // IP address on VirtualMachine's network interfaces included in the fields' CIDRs + // that will be used in respective status.addresses fields. + InternalNetworkSubnetCIDR string + ExternalNetworkSubnetCIDR string + // IP address on VirtualMachine's VM Network names that will be used to when searching + // for status.addresses fields. Note that if InternalNetworkSubnetCIDR and + // ExternalNetworkSubnetCIDR are not set, then the vNIC associated to this network must + // only have a single IP address assigned to it. + InternalVMNetworkName string + ExternalVMNetworkName string +} + +// CPIConfig is used to read and store information (related only to the CPI) from the cloud configuration file +type CPIConfig struct { + vcfg.Config + Nodes Nodes +} diff --git a/pkg/cloudprovider/vsphere/config/types_ini_legacy.go b/pkg/cloudprovider/vsphere/config/types_ini_legacy.go new file mode 100644 index 000000000..90974214b --- /dev/null +++ b/pkg/cloudprovider/vsphere/config/types_ini_legacy.go @@ -0,0 +1,46 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + vcfg "k8s.io/cloud-provider-vsphere/pkg/common/config" +) + +/* + TODO: + When the INI based cloud-config is deprecated. This file should be deleted. +*/ + +// NodesINI captures internal/external networks +type NodesINI struct { + // IP address on VirtualMachine's network interfaces included in the fields' CIDRs + // that will be used in respective status.addresses fields. + InternalNetworkSubnetCIDR string `gcfg:"internal-network-subnet-cidr"` + ExternalNetworkSubnetCIDR string `gcfg:"external-network-subnet-cidr"` + // IP address on VirtualMachine's VM Network names that will be used to when searching + // for status.addresses fields. Note that if InternalNetworkSubnetCIDR and + // ExternalNetworkSubnetCIDR are not set, then the vNIC associated to this network must + // only have a single IP address assigned to it. + InternalVMNetworkName string `gcfg:"internal-vm-network-name"` + ExternalVMNetworkName string `gcfg:"external-vm-network-name"` +} + +// CPIConfigINI is the INI representation +type CPIConfigINI struct { + vcfg.CommonConfigINI + Nodes NodesINI +} diff --git a/pkg/cloudprovider/vsphere/config/types_yaml.go b/pkg/cloudprovider/vsphere/config/types_yaml.go new file mode 100644 index 000000000..c71a93d41 --- /dev/null +++ b/pkg/cloudprovider/vsphere/config/types_yaml.go @@ -0,0 +1,50 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + vcfg "k8s.io/cloud-provider-vsphere/pkg/common/config" +) + +/* + TODO: + When the INI based cloud-config is deprecated, this file should be renamed + from types_yaml.go to types.go and the structs within this file should be named: + + ConfigYAML -> Config + NodesYAML -> Nodes +*/ + +// NodesYAML captures internal/external networks +type NodesYAML struct { + // IP address on VirtualMachine's network interfaces included in the fields' CIDRs + // that will be used in respective status.addresses fields. + InternalNetworkSubnetCIDR string `yaml:"internalNetworkSubnetCidr"` + ExternalNetworkSubnetCIDR string `yaml:"externalNetworkSubnetCidr"` + // IP address on VirtualMachine's VM Network names that will be used to when searching + // for status.addresses fields. Note that if InternalNetworkSubnetCIDR and + // ExternalNetworkSubnetCIDR are not set, then the vNIC associated to this network must + // only have a single IP address assigned to it. + InternalVMNetworkName string `yaml:"internalVmNetworkName"` + ExternalVMNetworkName string `yaml:"externalVmNetworkName"` +} + +// CPIConfigYAML is the YAML representation +type CPIConfigYAML struct { + vcfg.CommonConfigYAML + Nodes NodesYAML +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/README.md b/pkg/cloudprovider/vsphere/loadbalancer/README.md index 7557aaf3f..5fe09c937 100644 --- a/pkg/cloudprovider/vsphere/loadbalancer/README.md +++ b/pkg/cloudprovider/vsphere/loadbalancer/README.md @@ -24,7 +24,7 @@ connected to this load balancer service. ## Features The load balancer controller part of the vsphere cloud controller manager -is optional. If no `LoadBalancer` or `LoadBalancerClass` section is given +is optional. If no `loadBalancer` or `loadBalancerClass` section is given in the controller configuration loadbalancing support is disabled. If enabled the follwing feature are supported: @@ -41,7 +41,7 @@ service object is already (accidentally) gone. This load balancer controller supports the usage of multiple load balancer classes. Classes are preconfigured in the configuration file of the cloud controller manager. There may be an arbitrary set of such classes in a dedicated -setup. Every class may use another `IPPool` configured in NSX-T. +setup. Every class may use another `ipPool` configured in NSX-T. This supports the creation of load balancers in different visibility realms, for example an `*internet facing* or a *private* load balancer. The IPPools must be preconfigured in NSX-T. @@ -72,29 +72,31 @@ For TCP load balancers a health check will be generated. The controller manager requires dedicated entries in the cloud controller's configuration file: -```ini -[LoadBalancer] -ipPoolName = pool1 -lbServiceId = 4711 -size = SMALL -tcpAppProfileName = default-tcp-lb-app-profile -udpAppProfileName = default-udp-lb-app-profile -tags = {\"tag1\": \"value1\", \"tag2\": \"value 2\"} - -[LoadBalancerClass "public"] -ipPoolName = poolPublic - -[LoadBalancerClass "private"] -ipPoolName = poolPrivate - -[NSX-T] -user = admin -password = secret -host = nsxt-server -insecure-flag = false +```yaml +loadBalancer: + ipPoolName: pool1 + lbServiceId: 4711 + size: SMALL + tcpAppProfileName: default-tcp-lb-app-profile + udpAppProfileName: default-udp-lb-app-profile + tags: + tag1: value1 + tag2: value 2 + +loadBalancerClass: + public: + ipPoolName: poolPublic + private": + ipPoolName: poolPrivate + +nsxt: + user: admin + password": secret + host: nsxt-server + insecureFlag: false ``` -If the `LoadBalancer` section or at least one `LoadBalancerClass` section is +If the `loadBalancer` section or at least one `loadBalancerClass` section is given, the load balancer support of the vSphere cloud controller manager is enabled, otherwise it is disabled. @@ -112,12 +114,12 @@ the tags and string values. The tag scope `owner` can be used to overwrite the owner name using the controller's app name by default. -The `LoadBalancer` section defines an implicit default load balancer class. This +The `loadBalancer` section defines an implicit default load balancer class. This load balancer class is used if the service does not specify a dedicated load balancer class via annotation. Its values are also used as defaults for all explicitly specified load balancer classes. -Additionaly classes may be configured by the `LoadBalancerClass` +Additionaly classes may be configured by the `loadBalancerClass` subsections. ### Managing Modes @@ -132,7 +134,7 @@ There are two different modes the load balancer support can be used with: gateway must be specified, which is used for the segments the cluster nodes are connected to. The NSX-T load balancer service is only created if it is required. This saves resources if no kubernetes service of type - `LoadBalancer` is actually used. + `loadBalancer` is actually used. Exactly one of the properties `lbServiceId` or `tier1GatewayPath` must be specified if the load balancer support for the vSphere cloud controller @@ -141,20 +143,20 @@ manager is enabled. If the load balancer service should be managed by the controller (*managed* mode), the `tier1GatewayPath` must be set (`lbServiceId` must not be set in this case): -```ini -[LoadBalancer] -ipPoolName = pool1 -tier1GatewayPath = /infra/tier-1s/12345 -size = SMALL -tcpAppProfileName = default-tcp-lb-app-profile -udpAppProfileName = default-udp-lb-app-profile +```yaml +loadBalancer: + ipPoolName: pool1 + tier1GatewayPath: /infra/tier-1s/12345 + size: SMALL + tcpAppProfileName: default-tcp-lb-app-profile + udpAppProfileName: default-udp-lb-app-profile ... ``` ### Configuraton Option Reference -The load balancer configuration uses the sections `[NSX-T]`, `[LoadBalancer]` and -the subsections `[LoadBalancerClass ""]` +The load balancer configuration uses the sections `nsxt`, `loadBalancer` and +the subsections `loadBalancerClass` #### Section NSX-T @@ -165,16 +167,16 @@ The following attributes are supported: |Attribute|Meaning| |---------|-------| |`host`|NSXT-T host| -|`insecure-flag`|to be set to true if NSX-T uses locally signed cert without specifying a ca| -|`ca-file`|certificate authority for the server certificate for locally signed certificates | +|`insecureFlag`|to be set to true if NSX-T uses locally signed cert without specifying a ca| +|`caFile`|certificate authority for the server certificate for locally signed certificates | |`user`|user name (either password, access token or certificate based authentification must be specified)| |`password`|password in clear text for password based authentification| |`vmcAccessToken`|access token for token based authentification| |`vmcAuthHost`|verification host for token based authentification| -|`client-auth-cert-file`|client certificate for the certificate based authorization| -|`client-auth-key-file`|private key for the client certificate| +|`clientAuthCertFile`|client certificate for the certificate based authorization| +|`clientAuthKeyFile`|private key for the client certificate| -#### Section LoadBalancer +#### Section loadBalancer The load balancer section contains general settings and default settings for the load balancer classes. The following attributes are supported: @@ -193,7 +195,7 @@ dangling elements in the infrastructure originating from this controller manager If the cluster name option is not given, there will be no automated cleanup of dangling elements. -Additionally the attributes of a `LoadBalancerClass` can be specified here. These +Additionally the attributes of a `loadBalancerClass` can be specified here. These values are used as defaults for configured load balancer classes. If no explicit default load balancer class (with name `default`) is configured, these settings are used for the default load balancer class. @@ -202,7 +204,7 @@ The default load balancer class settings are always used if the kubernetes service object does not explicitly specify a load balancer class by using the annotation `loadbalancer.vmware.io/class`. -#### Subsections LoadBalancerClass +#### Subsections loadBalancerClass The name of the subsection is used as name for the load balancer class to configure. A load balancer class configuration uses the following attributes: @@ -216,7 +218,7 @@ A load balancer class configuration uses the following attributes: |`udpAppProfileName`| name of application profile used for UDP connections (either `udpAppProfileName` or `udpAppProfileID` must be specified)| |`udpAppProfileID`| id of application profile used for UDP connections| -If a name/id pair is missing completely it will be defaulted by the settings from the `LoadBalancer` section. +If a name/id pair is missing completely it will be defaulted by the settings from the `loadBalancer` section. If there no value is specified, also, the configuration is invalid. If a name is specified instead of an id there *MUST* not be multiple such elements with the same name, even this is possible in NSX-T. diff --git a/pkg/cloudprovider/vsphere/loadbalancer/class.go b/pkg/cloudprovider/vsphere/loadbalancer/class.go index bbd9876a7..8f7a86f1a 100644 --- a/pkg/cloudprovider/vsphere/loadbalancer/class.go +++ b/pkg/cloudprovider/vsphere/loadbalancer/class.go @@ -56,7 +56,7 @@ func setupClasses(access NSXTAccess, cfg *config.LBConfig) (*loadBalancerClasses if err != nil { return nil, errors.Wrapf(err, "invalid LoadBalancerClass %s", config.DefaultLoadBalancerClass) } - if defCfg, ok := cfg.LoadBalancerClasses[defaultClass.className]; ok { + if defCfg, ok := cfg.LoadBalancerClass[defaultClass.className]; ok { defaultClass, err = newLBClass(config.DefaultLoadBalancerClass, defCfg, defaultClass, resolver) if err != nil { return nil, errors.Wrapf(err, "invalid LoadBalancerClass %s", config.DefaultLoadBalancerClass) @@ -65,7 +65,7 @@ func setupClasses(access NSXTAccess, cfg *config.LBConfig) (*loadBalancerClasses lbClasses.add(defaultClass) } - for name, classConfig := range cfg.LoadBalancerClasses { + for name, classConfig := range cfg.LoadBalancerClass { if _, ok := lbClasses.classes[name]; ok { return nil, fmt.Errorf("duplicate LoadBalancerClass %s", name) } diff --git a/pkg/cloudprovider/vsphere/loadbalancer/config/config.go b/pkg/cloudprovider/vsphere/loadbalancer/config/config.go index 68b26b348..9e6dfaaab 100644 --- a/pkg/cloudprovider/vsphere/loadbalancer/config/config.go +++ b/pkg/cloudprovider/vsphere/loadbalancer/config/config.go @@ -17,126 +17,22 @@ package config import ( - "encoding/json" "fmt" - "io" "os" "strconv" - "strings" - "gopkg.in/gcfg.v1" - - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog" - - "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" -) - -const ( - // DefaultLoadBalancerClass is the default load balancer class - DefaultLoadBalancerClass = "default" ) -// LoadBalancerSizes contains the valid size names -var LoadBalancerSizes = sets.NewString( - model.LBService_SIZE_SMALL, - model.LBService_SIZE_MEDIUM, - model.LBService_SIZE_LARGE, - model.LBService_SIZE_XLARGE, - model.LBService_SIZE_DLB, -) - -// LBConfig is used to read and store information from the cloud configuration file -type LBConfig struct { - LoadBalancer LoadBalancerConfig `gcfg:"LoadBalancer"` - LoadBalancerClasses map[string]*LoadBalancerClassConfig `gcfg:"LoadBalancerClass"` - NSXT NsxtConfig `gcfg:"NSX-T"` -} - -// LoadBalancerConfig contains the configuration for the load balancer itself -type LoadBalancerConfig struct { - LoadBalancerClassConfig - Size string `gcfg:"size"` - LBServiceID string `gcfg:"lbServiceId"` - Tier1GatewayPath string `gcfg:"tier1GatewayPath"` - RawTags string `gcfg:"tags"` - AdditionalTags map[string]string -} - -// LoadBalancerClassConfig contains the configuration for a load balancer class -type LoadBalancerClassConfig struct { - IPPoolName string `gcfg:"ipPoolName"` - IPPoolID string `gcfg:"ipPoolID"` - TCPAppProfileName string `gcfg:"tcpAppProfileName"` - TCPAppProfilePath string `gcfg:"tcpAppProfilePath"` - UDPAppProfileName string `gcfg:"udpAppProfileName"` - UDPAppProfilePath string `gcfg:"udpAppProfilePath"` -} - -// NsxtConfig contains the NSX-T specific configuration -type NsxtConfig struct { - // NSX-T username. - User string `gcfg:"user"` - // NSX-T password in clear text. - Password string `gcfg:"password"` - // NSX-T host. - Host string `gcfg:"host"` - // InsecureFlag is to be set to true if NSX-T uses self-signed cert. - InsecureFlag bool `gcfg:"insecure-flag"` - - VMCAccessToken string `gcfg:"vmcAccessToken"` - VMCAuthHost string `gcfg:"vmcAuthHost"` - ClientAuthCertFile string `gcfg:"client-auth-cert-file"` - ClientAuthKeyFile string `gcfg:"client-auth-key-file"` - CAFile string `gcfg:"ca-file"` -} +/* + TODO: + When the INI based cloud-config is deprecated, this functions below should be preserved +*/ // IsEnabled checks whether the load balancer feature is enabled // It is enabled if any flavor of the load balancer configuration is given. func (cfg *LBConfig) IsEnabled() bool { - return len(cfg.LoadBalancerClasses) > 0 || !cfg.LoadBalancer.IsEmpty() -} - -func (cfg *LBConfig) validateConfig() error { - if cfg.LoadBalancer.LBServiceID == "" && cfg.LoadBalancer.Tier1GatewayPath == "" { - msg := "either load balancer service id or T1 gateway path required" - klog.Errorf(msg) - return fmt.Errorf(msg) - } - if cfg.LoadBalancer.TCPAppProfileName == "" && cfg.LoadBalancer.TCPAppProfilePath == "" { - msg := "either load balancer TCP application profile name or path required" - klog.Errorf(msg) - return fmt.Errorf(msg) - } - if cfg.LoadBalancer.UDPAppProfileName == "" && cfg.LoadBalancer.UDPAppProfilePath == "" { - msg := "either load balancer UDP application profile name or path required" - klog.Errorf(msg) - return fmt.Errorf(msg) - } - if !LoadBalancerSizes.Has(cfg.LoadBalancer.Size) { - msg := fmt.Sprintf("load balancer size is invalid. Valid values are: %s", strings.Join(LoadBalancerSizes.List(), ",")) - klog.Errorf(msg) - return fmt.Errorf(msg) - } - if cfg.LoadBalancer.IPPoolID == "" && cfg.LoadBalancer.IPPoolName == "" { - class, ok := cfg.LoadBalancerClasses[DefaultLoadBalancerClass] - if !ok { - msg := "no default load balancer class defined" - klog.Errorf(msg) - return fmt.Errorf(msg) - } else if class.IPPoolName == "" && class.IPPoolID == "" { - msg := "default load balancer class: ipPoolName and ipPoolID is empty" - klog.Errorf(msg) - return fmt.Errorf(msg) - } - } else { - if cfg.LoadBalancer.IPPoolName != "" && cfg.LoadBalancer.IPPoolID != "" { - msg := "either load balancer ipPoolName or ipPoolID can be set" - klog.Errorf(msg) - return fmt.Errorf(msg) - } - } - return cfg.NSXT.validateConfig() + return len(cfg.LoadBalancerClass) > 0 || !cfg.LoadBalancer.IsEmpty() } // IsEmpty checks whether the load balancer config is empty (no values specified) @@ -146,32 +42,6 @@ func (cfg *LoadBalancerConfig) IsEmpty() bool { cfg.Tier1GatewayPath == "" } -func (cfg *NsxtConfig) validateConfig() error { - if cfg.VMCAccessToken != "" { - if cfg.VMCAuthHost == "" { - msg := "vmc auth host must be provided if auth token is provided" - klog.Errorf(msg) - return fmt.Errorf(msg) - } - } else if cfg.User != "" { - if cfg.Password == "" { - msg := "password is empty" - klog.Errorf(msg) - return fmt.Errorf(msg) - } - } else { - msg := "either user or vmc access token must be set" - klog.Errorf(msg) - return fmt.Errorf(msg) - } - if cfg.Host == "" { - msg := "host is empty" - klog.Errorf(msg) - return fmt.Errorf(msg) - } - return nil -} - // FromEnv initializes the provided configuration object with values // obtained from environment variables. If an environment variable is set // for a property that's already initialized, the environment variable's value @@ -207,55 +77,40 @@ func (cfg *NsxtConfig) FromEnv() error { return nil } -// ReadConfig parses vSphere cloud config file and stores it into LBConfig. -// Environment variables are also checked -func ReadConfig(config io.Reader) (*LBConfig, error) { - if config == nil { - return nil, fmt.Errorf("no vSphere cloud provider config file given") - } - - cfg := &LBConfig{} +/* + TODO: + When the INI based cloud-config is deprecated, the references to the + INI based code (ie the call to ReadConfigINI) below should be deleted. +*/ - if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, config)); err != nil { - return nil, err +// ReadLBConfig parses vSphere cloud config file and stores it into VSphereConfig. +// Environment variables are also checked +func ReadLBConfig(byConfig []byte) (*LBConfig, error) { + if len(byConfig) == 0 { + return nil, fmt.Errorf("Invalid YAML/INI file") } - err := cfg.CompleteAndValidate() + cfg, err := ReadConfigYAML(byConfig) if err != nil { - return nil, err - } - return cfg, nil -} + klog.Warningf("ReadConfigYAML failed: %s", err) -// CompleteAndValidate sets default values, overrides by env and validates the resulting config -func (cfg *LBConfig) CompleteAndValidate() error { - if !cfg.IsEnabled() { - return nil - } - - cfg.LoadBalancer.AdditionalTags = map[string]string{} - if cfg.LoadBalancer.RawTags != "" { - err := json.Unmarshal([]byte(cfg.LoadBalancer.RawTags), &cfg.LoadBalancer.AdditionalTags) + cfg, err = ReadConfigINI(byConfig) if err != nil { - return fmt.Errorf("unmarshalling load balancer tags failed: %s", err) - } - } - if cfg.LoadBalancerClasses == nil { - cfg.LoadBalancerClasses = map[string]*LoadBalancerClassConfig{} - } - for _, class := range cfg.LoadBalancerClasses { - if class.IPPoolName == "" { - if class.IPPoolID == "" { - class.IPPoolID = cfg.LoadBalancer.IPPoolID - class.IPPoolName = cfg.LoadBalancer.IPPoolName - } + klog.Errorf("ReadConfigINI failed: %s", err) + return nil, err } + + klog.Info("ReadConfig INI succeeded. LoadBalancer INI-based cloud-config is deprecated and will be removed in 2.0. Please use YAML based cloud-config.") + } else { + klog.Info("ReadConfig YAML succeeded") } // Env Vars should override config file entries if present if err := cfg.NSXT.FromEnv(); err != nil { - return err + klog.Errorf("FromEnv failed: %s", err) + return nil, err } - return cfg.validateConfig() + klog.Info("Config initialized") + return cfg, nil } diff --git a/pkg/cloudprovider/vsphere/loadbalancer/config/config_ini_legacy.go b/pkg/cloudprovider/vsphere/loadbalancer/config/config_ini_legacy.go new file mode 100644 index 000000000..36feb2a6d --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/config/config_ini_legacy.go @@ -0,0 +1,217 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "encoding/json" + "fmt" + "strings" + + "gopkg.in/gcfg.v1" + + "k8s.io/klog" +) + +/* + TODO: + When the INI based cloud-config is deprecated. This file should be deleted. +*/ + +// CreateConfig generates a common Config object based on what other structs and funcs +// are already dependent upon in other packages. +func (lbc *LBConfigINI) CreateConfig() *LBConfig { + cfg := &LBConfig{ + LoadBalancerClass: make(map[string]*LoadBalancerClassConfig), + } + + //LoadBalancerClassConfig + cfg.LoadBalancer.IPPoolName = lbc.LoadBalancer.IPPoolName + cfg.LoadBalancer.IPPoolID = lbc.LoadBalancer.IPPoolID + cfg.LoadBalancer.TCPAppProfileName = lbc.LoadBalancer.TCPAppProfileName + cfg.LoadBalancer.TCPAppProfilePath = lbc.LoadBalancer.TCPAppProfilePath + cfg.LoadBalancer.UDPAppProfileName = lbc.LoadBalancer.UDPAppProfileName + cfg.LoadBalancer.UDPAppProfilePath = lbc.LoadBalancer.UDPAppProfilePath + //LoadBalancerClassConfig -> LoadBalancerConfig + cfg.LoadBalancer.Size = lbc.LoadBalancer.Size + cfg.LoadBalancer.LBServiceID = lbc.LoadBalancer.LBServiceID + cfg.LoadBalancer.Tier1GatewayPath = lbc.LoadBalancer.Tier1GatewayPath + cfg.LoadBalancer.AdditionalTags = lbc.LoadBalancer.AdditionalTags + + //LoadBalancerClass + for key, value := range lbc.LoadBalancerClass { + cfg.LoadBalancerClass[key] = &LoadBalancerClassConfig{ + IPPoolName: value.IPPoolName, + IPPoolID: value.IPPoolID, + TCPAppProfileName: value.TCPAppProfileName, + TCPAppProfilePath: value.TCPAppProfilePath, + UDPAppProfileName: value.UDPAppProfileName, + UDPAppProfilePath: value.UDPAppProfilePath, + } + } + + //NSXT + cfg.NSXT.User = lbc.NSXT.User + cfg.NSXT.Password = lbc.NSXT.Password + cfg.NSXT.Host = lbc.NSXT.Host + cfg.NSXT.InsecureFlag = lbc.NSXT.InsecureFlag + cfg.NSXT.VMCAccessToken = lbc.NSXT.VMCAccessToken + cfg.NSXT.VMCAuthHost = lbc.NSXT.VMCAuthHost + cfg.NSXT.ClientAuthCertFile = lbc.NSXT.ClientAuthCertFile + cfg.NSXT.ClientAuthKeyFile = lbc.NSXT.ClientAuthKeyFile + cfg.NSXT.CAFile = lbc.NSXT.CAFile + + return cfg +} + +func (lbc *LBConfigINI) isEnabled() bool { + return len(lbc.LoadBalancerClass) > 0 || !lbc.LoadBalancer.isEmpty() +} + +func (lbc *LBConfigINI) validateConfig() error { + if lbc.LoadBalancer.LBServiceID == "" && lbc.LoadBalancer.Tier1GatewayPath == "" { + msg := "either load balancer service id or T1 gateway path required" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if lbc.LoadBalancer.TCPAppProfileName == "" && lbc.LoadBalancer.TCPAppProfilePath == "" { + msg := "either load balancer TCP application profile name or path required" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if lbc.LoadBalancer.UDPAppProfileName == "" && lbc.LoadBalancer.UDPAppProfilePath == "" { + msg := "either load balancer UDP application profile name or path required" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if !LoadBalancerSizes.Has(lbc.LoadBalancer.Size) { + msg := fmt.Sprintf("load balancer size is invalid. Valid values are: %s", strings.Join(LoadBalancerSizes.List(), ",")) + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if lbc.LoadBalancer.IPPoolID == "" && lbc.LoadBalancer.IPPoolName == "" { + class, ok := lbc.LoadBalancerClass[DefaultLoadBalancerClass] + if !ok { + msg := "no default load balancer class defined" + klog.Errorf(msg) + return fmt.Errorf(msg) + } else if class.IPPoolName == "" && class.IPPoolID == "" { + msg := "default load balancer class: ipPoolName and ipPoolID is empty" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + } else { + if lbc.LoadBalancer.IPPoolName != "" && lbc.LoadBalancer.IPPoolID != "" { + msg := "either load balancer ipPoolName or ipPoolID can be set" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + } + return lbc.NSXT.validateConfig() +} + +func (lbc *LoadBalancerConfigINI) isEmpty() bool { + return lbc.Size == "" && lbc.LBServiceID == "" && + lbc.IPPoolID == "" && lbc.IPPoolName == "" && + lbc.Tier1GatewayPath == "" +} + +func (lbc *NsxtConfigINI) validateConfig() error { + if lbc.VMCAccessToken != "" { + if lbc.VMCAuthHost == "" { + msg := "vmc auth host must be provided if auth token is provided" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + } else if lbc.User != "" { + if lbc.Password == "" { + msg := "password is empty" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + } else { + msg := "either user or vmc access token must be set" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if lbc.Host == "" { + msg := "host is empty" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + return nil +} + +// CompleteAndValidate sets default values, overrides by env and validates the resulting config +func (lbc *LBConfigINI) CompleteAndValidate() error { + if !lbc.isEnabled() { + return nil + } + + lbc.LoadBalancer.AdditionalTags = map[string]string{} + if lbc.LoadBalancer.RawTags != "" { + err := json.Unmarshal([]byte(lbc.LoadBalancer.RawTags), &lbc.LoadBalancer.AdditionalTags) + if err != nil { + return fmt.Errorf("unmarshalling load balancer tags failed: %s", err) + } + } + if lbc.LoadBalancerClass == nil { + lbc.LoadBalancerClass = map[string]*LoadBalancerClassConfigINI{} + } + for _, class := range lbc.LoadBalancerClass { + if class.IPPoolName == "" { + class.IPPoolName = lbc.LoadBalancer.IPPoolName + } + if class.IPPoolID == "" { + class.IPPoolID = lbc.LoadBalancer.IPPoolID + } + } + + return lbc.validateConfig() +} + +// ReadRawConfigINI parses vSphere cloud config file and stores it into ConfigINI +func ReadRawConfigINI(byConfig []byte) (*LBConfigINI, error) { + if len(byConfig) == 0 { + return nil, fmt.Errorf("Invalid INI file") + } + + strConfig := string(byConfig[:]) + + cfg := &LBConfigINI{ + LoadBalancerClass: make(map[string]*LoadBalancerClassConfigINI), + } + + if err := gcfg.FatalOnly(gcfg.ReadStringInto(cfg, strConfig)); err != nil { + return nil, err + } + + err := cfg.CompleteAndValidate() + if err != nil { + return nil, err + } + return cfg, nil +} + +// ReadConfigINI parses vSphere cloud config file and stores it into Config +func ReadConfigINI(byConfig []byte) (*LBConfig, error) { + cfg, err := ReadRawConfigINI(byConfig) + if err != nil { + return nil, err + } + + return cfg.CreateConfig(), nil +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/config/config_test.go b/pkg/cloudprovider/vsphere/loadbalancer/config/config_ini_legacy_test.go similarity index 57% rename from pkg/cloudprovider/vsphere/loadbalancer/config/config_test.go rename to pkg/cloudprovider/vsphere/loadbalancer/config/config_ini_legacy_test.go index 0fd9dbd2d..20a4c6994 100644 --- a/pkg/cloudprovider/vsphere/loadbalancer/config/config_test.go +++ b/pkg/cloudprovider/vsphere/loadbalancer/config/config_ini_legacy_test.go @@ -17,35 +17,39 @@ package config import ( - "strings" "testing" ) -func TestReadConfig(t *testing.T) { +/* + TODO: + When the INI based cloud-config is deprecated. This file should be deleted. +*/ + +func TestReadINIConfig(t *testing.T) { contents := ` [LoadBalancer] -ipPoolName = pool1 +ip-pool-name = pool1 size = MEDIUM -lbServiceId = 4711 -tier1GatewayPath = 1234 -tcpAppProfileName = default-tcp-lb-app-profile -udpAppProfileName = default-udp-lb-app-profile +lb-service-id = 4711 +tier1-gateway-path = 1234 +tcp-app-profile-name = default-tcp-lb-app-profile +udp-app-profile-name = default-udp-lb-app-profile tags = {\"tag1\": \"value1\", \"tag2\": \"value 2\"} [LoadBalancerClass "public"] -ipPoolName = poolPublic +ip-pool-name = poolPublic [LoadBalancerClass "private"] -ipPoolName = poolPrivate -tcpAppProfileName = tcp2 -udpAppProfileName = udp2 +ip-pool-name = poolPrivate +tcp-app-profile-name = tcp2 +udp-app-profile-name = udp2 -[NSX-T] +[NSXT] user = admin password = secret host = nsxt-server ` - config, err := ReadConfig(strings.NewReader(contents)) + config, err := ReadRawConfigINI([]byte(contents)) if err != nil { t.Error(err) return @@ -62,36 +66,36 @@ host = nsxt-server assertEquals("LoadBalancer.tcpAppProfileName", config.LoadBalancer.TCPAppProfileName, "default-tcp-lb-app-profile") assertEquals("LoadBalancer.udpAppProfileName", config.LoadBalancer.UDPAppProfileName, "default-udp-lb-app-profile") assertEquals("LoadBalancer.size", config.LoadBalancer.Size, "MEDIUM") - if len(config.LoadBalancerClasses) != 2 { - t.Errorf("expected two LoadBalancerClass subsections, but got %d", len(config.LoadBalancerClasses)) + if len(config.LoadBalancerClass) != 2 { + t.Errorf("expected two LoadBalancerClass subsections, but got %d", len(config.LoadBalancerClass)) } - assertEquals("LoadBalancerClass.public.ipPoolName", config.LoadBalancerClasses["public"].IPPoolName, "poolPublic") - assertEquals("LoadBalancerClass.private.tcpAppProfileName", config.LoadBalancerClasses["private"].TCPAppProfileName, "tcp2") - assertEquals("LoadBalancerClass.private.udpAppProfileName", config.LoadBalancerClasses["private"].UDPAppProfileName, "udp2") + assertEquals("LoadBalancerClass.public.ipPoolName", config.LoadBalancerClass["public"].IPPoolName, "poolPublic") + assertEquals("LoadBalancerClass.private.tcpAppProfileName", config.LoadBalancerClass["private"].TCPAppProfileName, "tcp2") + assertEquals("LoadBalancerClass.private.udpAppProfileName", config.LoadBalancerClass["private"].UDPAppProfileName, "udp2") if len(config.LoadBalancer.AdditionalTags) != 2 || config.LoadBalancer.AdditionalTags["tag1"] != "value1" || config.LoadBalancer.AdditionalTags["tag2"] != "value 2" { t.Errorf("unexpected additionalTags %v", config.LoadBalancer.AdditionalTags) } - assertEquals("NSX-T.user", config.NSXT.User, "admin") - assertEquals("NSX-T.password", config.NSXT.Password, "secret") - assertEquals("NSX-T.host", config.NSXT.Host, "nsxt-server") + assertEquals("NSXT.user", config.NSXT.User, "admin") + assertEquals("NSXT.password", config.NSXT.Password, "secret") + assertEquals("NSXT.host", config.NSXT.Host, "nsxt-server") } -func TestReadConfigOnVMC(t *testing.T) { +func TestReadINIConfigOnVMC(t *testing.T) { contents := ` [LoadBalancer] -ipPoolID = 123-456 +ip-pool-id = 123-456 size = MEDIUM -tier1GatewayPath = 1234 -tcpAppProfilePath = infra/xxx/tcp1234 -udpAppProfilePath = infra/xxx/udp1234 +tier1-gateway-path = 1234 +tcp-app-profile-path = infra/xxx/tcp1234 +udp-app-profile-path = infra/xxx/udp1234 -[NSX-T] -vmcAccessToken = token123 -vmcAuthHost = authHost +[NSXT] +vmc-access-token = token123 +vmc-auth-host = authHost host = nsxt-server insecure-flag = true ` - config, err := ReadConfig(strings.NewReader(contents)) + config, err := ReadRawConfigINI([]byte(contents)) if err != nil { t.Error(err) return @@ -101,15 +105,15 @@ insecure-flag = true t.Errorf("%s %s != %s", name, left, right) } } - assertEquals("LoadBalancer.ipPoolID", config.LoadBalancer.IPPoolID, "123-456") + assertEquals("LoadBalancer.ip-pool-id", config.LoadBalancer.IPPoolID, "123-456") assertEquals("LoadBalancer.size", config.LoadBalancer.Size, "MEDIUM") - assertEquals("LoadBalancer.tier1GatewayPath", config.LoadBalancer.Tier1GatewayPath, "1234") - assertEquals("LoadBalancer.tcpAppProfilePath", config.LoadBalancer.TCPAppProfilePath, "infra/xxx/tcp1234") - assertEquals("LoadBalancer.udpAppProfilePath", config.LoadBalancer.UDPAppProfilePath, "infra/xxx/udp1234") - assertEquals("NSX-T.vmcAccessToken", config.NSXT.VMCAccessToken, "token123") - assertEquals("NSX-T.vmcAuthHost", config.NSXT.VMCAuthHost, "authHost") - assertEquals("NSX-T.host", config.NSXT.Host, "nsxt-server") + assertEquals("LoadBalancer.tier1-gateway-path", config.LoadBalancer.Tier1GatewayPath, "1234") + assertEquals("LoadBalancer.tcp-app-profile-path", config.LoadBalancer.TCPAppProfilePath, "infra/xxx/tcp1234") + assertEquals("LoadBalancer.udp-app-profile-path", config.LoadBalancer.UDPAppProfilePath, "infra/xxx/udp1234") + assertEquals("NSXT.vmc-access-token", config.NSXT.VMCAccessToken, "token123") + assertEquals("NSXT.vmc-auth-host", config.NSXT.VMCAuthHost, "authHost") + assertEquals("NSXT.host", config.NSXT.Host, "nsxt-server") if !config.NSXT.InsecureFlag { - t.Errorf("NSX-T.insecure-flag != true") + t.Errorf("NSXT.insecure-flag != true") } } diff --git a/pkg/cloudprovider/vsphere/loadbalancer/config/config_yaml.go b/pkg/cloudprovider/vsphere/loadbalancer/config/config_yaml.go new file mode 100644 index 000000000..f726a403e --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/config/config_yaml.go @@ -0,0 +1,208 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "fmt" + "strings" + + yaml "gopkg.in/yaml.v2" + "k8s.io/klog" +) + +/* + TODO: + When the INI based cloud-config is deprecated, this file should be merged into config.go + and this file should be deleted. +*/ + +// CreateConfig generates a common Config object based on what other structs and funcs +// are already dependent upon in other packages. +func (lbc *LBConfigYAML) CreateConfig() *LBConfig { + cfg := &LBConfig{ + LoadBalancerClass: make(map[string]*LoadBalancerClassConfig), + } + + //LoadBalancerClassConfig + cfg.LoadBalancer.IPPoolName = lbc.LoadBalancer.IPPoolName + cfg.LoadBalancer.IPPoolID = lbc.LoadBalancer.IPPoolID + cfg.LoadBalancer.TCPAppProfileName = lbc.LoadBalancer.TCPAppProfileName + cfg.LoadBalancer.TCPAppProfilePath = lbc.LoadBalancer.TCPAppProfilePath + cfg.LoadBalancer.UDPAppProfileName = lbc.LoadBalancer.UDPAppProfileName + cfg.LoadBalancer.UDPAppProfilePath = lbc.LoadBalancer.UDPAppProfilePath + //LoadBalancerClassConfig -> LoadBalancerConfig + cfg.LoadBalancer.Size = lbc.LoadBalancer.Size + cfg.LoadBalancer.LBServiceID = lbc.LoadBalancer.LBServiceID + cfg.LoadBalancer.Tier1GatewayPath = lbc.LoadBalancer.Tier1GatewayPath + cfg.LoadBalancer.AdditionalTags = lbc.LoadBalancer.AdditionalTags + + //LoadBalancerClass + for key, value := range lbc.LoadBalancerClass { + cfg.LoadBalancerClass[key] = &LoadBalancerClassConfig{ + IPPoolName: value.IPPoolName, + IPPoolID: value.IPPoolID, + TCPAppProfileName: value.TCPAppProfileName, + TCPAppProfilePath: value.TCPAppProfilePath, + UDPAppProfileName: value.UDPAppProfileName, + UDPAppProfilePath: value.UDPAppProfilePath, + } + } + + //NSXT + cfg.NSXT.User = lbc.NSXT.User + cfg.NSXT.Password = lbc.NSXT.Password + cfg.NSXT.Host = lbc.NSXT.Host + cfg.NSXT.InsecureFlag = lbc.NSXT.InsecureFlag + cfg.NSXT.VMCAccessToken = lbc.NSXT.VMCAccessToken + cfg.NSXT.VMCAuthHost = lbc.NSXT.VMCAuthHost + cfg.NSXT.ClientAuthCertFile = lbc.NSXT.ClientAuthCertFile + cfg.NSXT.ClientAuthKeyFile = lbc.NSXT.ClientAuthKeyFile + cfg.NSXT.CAFile = lbc.NSXT.CAFile + + return cfg +} + +func (lbc *LBConfigYAML) isEnabled() bool { + return len(lbc.LoadBalancerClass) > 0 || !lbc.LoadBalancer.isEmpty() +} + +func (lbc *LBConfigYAML) validateConfig() error { + if lbc.LoadBalancer.LBServiceID == "" && lbc.LoadBalancer.Tier1GatewayPath == "" { + msg := "either load balancer service id or T1 gateway path required" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if lbc.LoadBalancer.TCPAppProfileName == "" && lbc.LoadBalancer.TCPAppProfilePath == "" { + msg := "either load balancer TCP application profile name or path required" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if lbc.LoadBalancer.UDPAppProfileName == "" && lbc.LoadBalancer.UDPAppProfilePath == "" { + msg := "either load balancer UDP application profile name or path required" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if !LoadBalancerSizes.Has(lbc.LoadBalancer.Size) { + msg := fmt.Sprintf("load balancer size is invalid. Valid values are: %s", strings.Join(LoadBalancerSizes.List(), ",")) + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if lbc.LoadBalancer.IPPoolID == "" && lbc.LoadBalancer.IPPoolName == "" { + class, ok := lbc.LoadBalancerClass[DefaultLoadBalancerClass] + if !ok { + msg := "no default load balancer class defined" + klog.Errorf(msg) + return fmt.Errorf(msg) + } else if class.IPPoolName == "" && class.IPPoolID == "" { + msg := "default load balancer class: ipPoolName and ipPoolID is empty" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + } else { + if lbc.LoadBalancer.IPPoolName != "" && lbc.LoadBalancer.IPPoolID != "" { + msg := "either load balancer ipPoolName or ipPoolID can be set" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + } + return lbc.NSXT.validateConfig() +} + +func (lbc *LoadBalancerConfigYAML) isEmpty() bool { + return lbc.Size == "" && lbc.LBServiceID == "" && + lbc.IPPoolID == "" && lbc.IPPoolName == "" && + lbc.Tier1GatewayPath == "" +} + +func (lbc *NsxtConfigYAML) validateConfig() error { + if lbc.VMCAccessToken != "" { + if lbc.VMCAuthHost == "" { + msg := "vmc auth host must be provided if auth token is provided" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + } else if lbc.User != "" { + if lbc.Password == "" { + msg := "password is empty" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + } else { + msg := "either user or vmc access token must be set" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + if lbc.Host == "" { + msg := "host is empty" + klog.Errorf(msg) + return fmt.Errorf(msg) + } + return nil +} + +// CompleteAndValidate sets default values, overrides by env and validates the resulting config +func (lbc *LBConfigYAML) CompleteAndValidate() error { + if !lbc.isEnabled() { + return nil + } + + if lbc.LoadBalancerClass == nil { + lbc.LoadBalancerClass = map[string]*LoadBalancerClassConfigYAML{} + } + for _, class := range lbc.LoadBalancerClass { + if class.IPPoolName == "" { + class.IPPoolName = lbc.LoadBalancer.IPPoolName + } + if class.IPPoolID == "" { + class.IPPoolID = lbc.LoadBalancer.IPPoolID + } + } + + return lbc.validateConfig() +} + +// ReadRawConfigYAML parses vSphere cloud config file and stores it into ConfigYAML +func ReadRawConfigYAML(byConfig []byte) (*LBConfigYAML, error) { + if len(byConfig) == 0 { + return nil, fmt.Errorf("Invalid YAML file") + } + + cfg := LBConfigYAML{ + LoadBalancerClass: make(map[string]*LoadBalancerClassConfigYAML), + } + + if err := yaml.Unmarshal(byConfig, &cfg); err != nil { + klog.Errorf("Unmarshal failed: %s", err) + return nil, err + } + + err := cfg.CompleteAndValidate() + if err != nil { + return nil, err + } + return &cfg, nil +} + +// ReadConfigYAML parses vSphere cloud config file and stores it into Config +func ReadConfigYAML(byConfig []byte) (*LBConfig, error) { + cfg, err := ReadRawConfigYAML(byConfig) + if err != nil { + return nil, err + } + + return cfg.CreateConfig(), nil +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/config/config_yaml_test.go b/pkg/cloudprovider/vsphere/loadbalancer/config/config_yaml_test.go new file mode 100644 index 000000000..9a7857f6c --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/config/config_yaml_test.go @@ -0,0 +1,121 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "testing" +) + +/* + TODO: + When the INI based cloud-config is deprecated. This file should be deleted. +*/ + +func TestReadYAMLConfig(t *testing.T) { + contents := ` +loadBalancer: + ipPoolName: pool1 + size: MEDIUM + lbServiceId: 4711 + tier1GatewayPath: 1234 + tcpAppProfileName: default-tcp-lb-app-profile + udpAppProfileName: default-udp-lb-app-profile + tags: + tag1: value1 + tag2: value 2 + +loadBalancerClass: + public: + ipPoolName: poolPublic + private: + ipPoolName: poolPrivate + tcpAppProfileName: tcp2 + udpAppProfileName: udp2 + +nsxt: + user: admin + password: secret + host: nsxt-server +` + config, err := ReadRawConfigYAML([]byte(contents)) + if err != nil { + t.Error(err) + return + } + + assertEquals := func(name, left, right string) { + if left != right { + t.Errorf("%s %s != %s", name, left, right) + } + } + assertEquals("loadBalancer.ipPoolName", config.LoadBalancer.IPPoolName, "pool1") + assertEquals("loadBalancer.lbServiceId", config.LoadBalancer.LBServiceID, "4711") + assertEquals("loadBalancer.tier1GatewayPath", config.LoadBalancer.Tier1GatewayPath, "1234") + assertEquals("loadBalancer.tcpAppProfileName", config.LoadBalancer.TCPAppProfileName, "default-tcp-lb-app-profile") + assertEquals("loadBalancer.udpAppProfileName", config.LoadBalancer.UDPAppProfileName, "default-udp-lb-app-profile") + assertEquals("loadBalancer.size", config.LoadBalancer.Size, "MEDIUM") + if len(config.LoadBalancerClass) != 2 { + t.Errorf("expected two LoadBalancerClass subsections, but got %d", len(config.LoadBalancerClass)) + } + assertEquals("loadBalancerClass.public.ipPoolName", config.LoadBalancerClass["public"].IPPoolName, "poolPublic") + assertEquals("loadBalancerClass.private.tcpAppProfileName", config.LoadBalancerClass["private"].TCPAppProfileName, "tcp2") + assertEquals("loadBalancerClass.private.udpAppProfileName", config.LoadBalancerClass["private"].UDPAppProfileName, "udp2") + if len(config.LoadBalancer.AdditionalTags) != 2 || config.LoadBalancer.AdditionalTags["tag1"] != "value1" || config.LoadBalancer.AdditionalTags["tag2"] != "value 2" { + t.Errorf("unexpected additionalTags %v", config.LoadBalancer.AdditionalTags) + } + assertEquals("nsxt.user", config.NSXT.User, "admin") + assertEquals("nsxt.password", config.NSXT.Password, "secret") + assertEquals("nsxt.host", config.NSXT.Host, "nsxt-server") +} + +func TestReadYAMLConfigOnVMC(t *testing.T) { + contents := ` +loadBalancer: + ipPoolId: 123-456 + size: MEDIUM + tier1GatewayPath: 1234 + tcpAppProfilePath: infra/xxx/tcp1234 + udpAppProfilePath: infra/xxx/udp1234 + +nsxt: + vmcAccessToken: token123 + vmcAuthHost: authHost + host: nsxt-server + insecureFlag: true +` + config, err := ReadRawConfigYAML([]byte(contents)) + if err != nil { + t.Error(err) + return + } + assertEquals := func(name, left, right string) { + if left != right { + t.Errorf("%s %s != %s", name, left, right) + } + } + assertEquals("loadBalancer.ipPoolId", config.LoadBalancer.IPPoolID, "123-456") + assertEquals("loadBalancer.size", config.LoadBalancer.Size, "MEDIUM") + assertEquals("loadBalancer.tier1GatewayPath", config.LoadBalancer.Tier1GatewayPath, "1234") + assertEquals("loadBalancer.tcpAppProfilePath", config.LoadBalancer.TCPAppProfilePath, "infra/xxx/tcp1234") + assertEquals("loadBalancer.udpAppProfilePath", config.LoadBalancer.UDPAppProfilePath, "infra/xxx/udp1234") + assertEquals("nsxt.vmcAccessToken", config.NSXT.VMCAccessToken, "token123") + assertEquals("nsxt.vmcAuthHost", config.NSXT.VMCAuthHost, "authHost") + assertEquals("nsxt.host", config.NSXT.Host, "nsxt-server") + if !config.NSXT.InsecureFlag { + t.Errorf("nsxt.insecureFlag != true") + } +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/config/consts_and_errors.go b/pkg/cloudprovider/vsphere/loadbalancer/config/consts_and_errors.go new file mode 100644 index 000000000..d54ed9a96 --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/config/consts_and_errors.go @@ -0,0 +1,37 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +const ( + // DefaultLoadBalancerClass is the default load balancer class + DefaultLoadBalancerClass = "default" +) + +// LoadBalancerSizes contains the valid size names +var LoadBalancerSizes = sets.NewString( + model.LBService_SIZE_SMALL, + model.LBService_SIZE_MEDIUM, + model.LBService_SIZE_LARGE, + model.LBService_SIZE_XLARGE, + model.LBService_SIZE_DLB, +) diff --git a/pkg/cloudprovider/vsphere/loadbalancer/config/types_common.go b/pkg/cloudprovider/vsphere/loadbalancer/config/types_common.go new file mode 100644 index 000000000..303de9e9d --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/config/types_common.go @@ -0,0 +1,61 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +// LBConfig is used to read and store information from the cloud configuration file +type LBConfig struct { + LoadBalancer LoadBalancerConfig + LoadBalancerClass map[string]*LoadBalancerClassConfig + NSXT NsxtConfig +} + +// LoadBalancerConfig contains the configuration for the load balancer itself +type LoadBalancerConfig struct { + LoadBalancerClassConfig + Size string + LBServiceID string + Tier1GatewayPath string + AdditionalTags map[string]string +} + +// LoadBalancerClassConfig contains the configuration for a load balancer class +type LoadBalancerClassConfig struct { + IPPoolName string + IPPoolID string + TCPAppProfileName string + TCPAppProfilePath string + UDPAppProfileName string + UDPAppProfilePath string +} + +// NsxtConfig contains the NSX-T specific configuration +type NsxtConfig struct { + // NSX-T username. + User string + // NSX-T password in clear text. + Password string + // NSX-T host. + Host string + // InsecureFlag is to be set to true if NSX-T uses self-signed cert. + InsecureFlag bool + + VMCAccessToken string + VMCAuthHost string + ClientAuthCertFile string + ClientAuthKeyFile string + CAFile string +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/config/types_ini_legacy.go b/pkg/cloudprovider/vsphere/loadbalancer/config/types_ini_legacy.go new file mode 100644 index 000000000..d8a68b929 --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/config/types_ini_legacy.go @@ -0,0 +1,62 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +// LBConfigINI is used to read and store information from the cloud configuration file +type LBConfigINI struct { + LoadBalancer LoadBalancerConfigINI `gcfg:"loadbalancer"` + LoadBalancerClass map[string]*LoadBalancerClassConfigINI `gcfg:"loadbalancerclass"` + NSXT NsxtConfigINI `gcfg:"nsxt"` +} + +// LoadBalancerConfigINI contains the configuration for the load balancer itself +type LoadBalancerConfigINI struct { + LoadBalancerClassConfigINI + Size string `gcfg:"size"` + LBServiceID string `gcfg:"lb-service-id"` + Tier1GatewayPath string `gcfg:"tier1-gateway-path"` + RawTags string `gcfg:"tags"` + AdditionalTags map[string]string +} + +// LoadBalancerClassConfigINI contains the configuration for a load balancer class +type LoadBalancerClassConfigINI struct { + IPPoolName string `gcfg:"ip-pool-name"` + IPPoolID string `gcfg:"ip-pool-id"` + TCPAppProfileName string `gcfg:"tcp-app-profile-name"` + TCPAppProfilePath string `gcfg:"tcp-app-profile-path"` + UDPAppProfileName string `gcfg:"udp-app-profile-name"` + UDPAppProfilePath string `gcfg:"udp-app-profile-path"` +} + +// NsxtConfigINI contains the NSX-T specific configuration +type NsxtConfigINI struct { + // NSX-T username. + User string `gcfg:"user"` + // NSX-T password in clear text. + Password string `gcfg:"password"` + // NSX-T host. + Host string `gcfg:"host"` + // InsecureFlag is to be set to true if NSX-T uses self-signed cert. + InsecureFlag bool `gcfg:"insecure-flag"` + + VMCAccessToken string `gcfg:"vmc-access-token"` + VMCAuthHost string `gcfg:"vmc-auth-host"` + ClientAuthCertFile string `gcfg:"client-auth-cert-file"` + ClientAuthKeyFile string `gcfg:"client-auth-key-file"` + CAFile string `gcfg:"ca-file"` +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/config/types_yaml.go b/pkg/cloudprovider/vsphere/loadbalancer/config/types_yaml.go new file mode 100644 index 000000000..f64cd5869 --- /dev/null +++ b/pkg/cloudprovider/vsphere/loadbalancer/config/types_yaml.go @@ -0,0 +1,80 @@ +/* + Copyright 2020 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +/* + TODO: + When the INI based cloud-config is deprecated, this file should be renamed + from types_yaml.go to types.go and the structs within this file should be named: + + LBConfigYAML -> LBConfig + LoadBalancerClassConfigYAML -> LoadBalancerClassConfig + LoadBalancerClassConfigYAML -> LoadBalancerClassConfig + NsxtConfigYAML -> NsxtConfig +*/ + +// LBConfigYAML is used to read and store information from the cloud configuration file +type LBConfigYAML struct { + LoadBalancer LoadBalancerConfigYAML `yaml:"loadBalancer"` + LoadBalancerClass map[string]*LoadBalancerClassConfigYAML `yaml:"loadBalancerClass"` + NSXT NsxtConfigYAML `yaml:"nsxt"` +} + +// LoadBalancerConfigYAML contains the configuration for the load balancer itself +type LoadBalancerConfigYAML struct { + Size string `yaml:"size"` + LBServiceID string `yaml:"lbServiceId"` + Tier1GatewayPath string `yaml:"tier1GatewayPath"` + AdditionalTags map[string]string `yaml:"tags"` + + // this struct use to inherit from LoadBalancerClassConfigYAML, but the YAML parser + // wasnt able to indirectly parse inherited fields + IPPoolName string `yaml:"ipPoolName"` + IPPoolID string `yaml:"ipPoolId"` + TCPAppProfileName string `yaml:"tcpAppProfileName"` + TCPAppProfilePath string `yaml:"tcpAppProfilePath"` + UDPAppProfileName string `yaml:"udpAppProfileName"` + UDPAppProfilePath string `yaml:"udpAppProfilePath"` +} + +// LoadBalancerClassConfigYAML contains the configuration for a load balancer class +type LoadBalancerClassConfigYAML struct { + IPPoolName string `yaml:"ipPoolName"` + IPPoolID string `yaml:"ipPoolId"` + TCPAppProfileName string `yaml:"tcpAppProfileName"` + TCPAppProfilePath string `yaml:"tcpAppProfilePath"` + UDPAppProfileName string `yaml:"udpAppProfileName"` + UDPAppProfilePath string `yaml:"udpAppProfilePath"` +} + +// NsxtConfigYAML contains the NSX-T specific configuration +type NsxtConfigYAML struct { + // NSX-T username. + User string `yaml:"user"` + // NSX-T password in clear text. + Password string `yaml:"password"` + // NSX-T host. + Host string `yaml:"host"` + // InsecureFlag is to be set to true if NSX-T uses self-signed cert. + InsecureFlag bool `yaml:"insecureFlag"` + + VMCAccessToken string `yaml:"vmcAccessToken"` + VMCAuthHost string `yaml:"vmcAuthHost"` + ClientAuthCertFile string `yaml:"clientAuthCertFile"` + ClientAuthKeyFile string `yaml:"clientAuthKeyFile"` + CAFile string `yaml:"caFile"` +} diff --git a/pkg/cloudprovider/vsphere/loadbalancer/lbprovider.go b/pkg/cloudprovider/vsphere/loadbalancer/lbprovider.go index 184e4f4aa..32be898ae 100644 --- a/pkg/cloudprovider/vsphere/loadbalancer/lbprovider.go +++ b/pkg/cloudprovider/vsphere/loadbalancer/lbprovider.go @@ -53,6 +53,9 @@ var _ LBProvider = &lbProvider{} // NewLBProvider creates a new LBProvider func NewLBProvider(cfg *config.LBConfig) (LBProvider, error) { + if cfg == nil { + return nil, nil + } if !cfg.IsEnabled() { return nil, nil } diff --git a/pkg/cloudprovider/vsphere/nodemanager.go b/pkg/cloudprovider/vsphere/nodemanager.go index 21f798905..ccb11db4c 100644 --- a/pkg/cloudprovider/vsphere/nodemanager.go +++ b/pkg/cloudprovider/vsphere/nodemanager.go @@ -24,6 +24,7 @@ import ( "strings" v1 "k8s.io/api/core/v1" + ccfg "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/config" pb "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/proto" vcfg "k8s.io/cloud-provider-vsphere/pkg/common/config" cm "k8s.io/cloud-provider-vsphere/pkg/common/connectionmanager" @@ -47,14 +48,14 @@ var ( ErrVMNotFound = errors.New("VM not found") ) -func newNodeManager(cpiCfg *CPIConfig, cm *cm.ConnectionManager) *NodeManager { +func newNodeManager(cfg *ccfg.CPIConfig, cm *cm.ConnectionManager) *NodeManager { return &NodeManager{ nodeNameMap: make(map[string]*NodeInfo), nodeUUIDMap: make(map[string]*NodeInfo), nodeRegUUIDMap: make(map[string]*v1.Node), vcList: make(map[string]*VCenterInfo), connectionManager: cm, - cpiCfg: cpiCfg, + cfg: cfg, } } @@ -203,21 +204,21 @@ func (nm *NodeManager) DiscoverNode(nodeID string, searchBy cm.FindVM) error { var internalVMNetworkName string var externalVMNetworkName string - if nm.cpiCfg != nil { - if nm.cpiCfg.Nodes.InternalNetworkSubnetCIDR != "" { - _, internalNetworkSubnet, err = net.ParseCIDR(nm.cpiCfg.Nodes.InternalNetworkSubnetCIDR) + if nm.cfg != nil { + if nm.cfg.Nodes.InternalNetworkSubnetCIDR != "" { + _, internalNetworkSubnet, err = net.ParseCIDR(nm.cfg.Nodes.InternalNetworkSubnetCIDR) if err != nil { return err } } - if nm.cpiCfg.Nodes.ExternalNetworkSubnetCIDR != "" { - _, externalNetworkSubnet, err = net.ParseCIDR(nm.cpiCfg.Nodes.ExternalNetworkSubnetCIDR) + if nm.cfg.Nodes.ExternalNetworkSubnetCIDR != "" { + _, externalNetworkSubnet, err = net.ParseCIDR(nm.cfg.Nodes.ExternalNetworkSubnetCIDR) if err != nil { return err } } - internalVMNetworkName = nm.cpiCfg.Nodes.InternalVMNetworkName - externalVMNetworkName = nm.cpiCfg.Nodes.ExternalVMNetworkName + internalVMNetworkName = nm.cfg.Nodes.InternalVMNetworkName + externalVMNetworkName = nm.cfg.Nodes.ExternalVMNetworkName } var addressMatchingEnabled bool diff --git a/pkg/cloudprovider/vsphere/types.go b/pkg/cloudprovider/vsphere/types.go index 778c26c52..11246dce9 100644 --- a/pkg/cloudprovider/vsphere/types.go +++ b/pkg/cloudprovider/vsphere/types.go @@ -22,9 +22,9 @@ import ( v1 "k8s.io/api/core/v1" cloudprovider "k8s.io/cloud-provider" + ccfg "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/config" "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/loadbalancer" lbcfg "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/loadbalancer/config" - vcfg "k8s.io/cloud-provider-vsphere/pkg/common/config" cm "k8s.io/cloud-provider-vsphere/pkg/common/connectionmanager" k8s "k8s.io/cloud-provider-vsphere/pkg/common/kubernetes" "k8s.io/cloud-provider-vsphere/pkg/common/vclib" @@ -35,35 +35,30 @@ type GRPCServer interface { Start() } -// CPIConfig is used to read and store information (related only to the CPI) from the cloud configuration file -type CPIConfig struct { - vcfg.Config - lbcfg.LBConfig - - Nodes struct { - // IP address on VirtualMachine's network interfaces included in the fields' CIDRs - // that will be used in respective status.addresses fields. - InternalNetworkSubnetCIDR string `gcfg:"internal-network-subnet-cidr"` - ExternalNetworkSubnetCIDR string `gcfg:"external-network-subnet-cidr"` - // IP address on VirtualMachine's VM Network names that will be used to when searching - // for status.addresses fields. Note that if InternalNetworkSubnetCIDR and - // ExternalNetworkSubnetCIDR are not set, then the vNIC associated to this network must - // only have a single IP address assigned to it. - InternalVMNetworkName string `gcfg:"internal-vm-network-name"` - ExternalVMNetworkName string `gcfg:"external-vm-network-name"` - } -} - // VSphere is an implementation of cloud provider Interface for VSphere. type VSphere struct { - cfg *CPIConfig + // input (aka configs) and output (aka interfaces) + cfg *ccfg.CPIConfig + server GRPCServer + + /* + Interfaces start + */ + // pluggable interfaces (tbd) + cfgLB *lbcfg.LBConfig + loadbalancer loadbalancer.LBProvider + + // cloud provider interfaces + instances cloudprovider.Instances + zones cloudprovider.Zones + /* + Interfaces end + */ + + // internal plumbing connectionManager *cm.ConnectionManager nodeManager *NodeManager informMgr *k8s.InformerManager - loadbalancer loadbalancer.LBProvider - instances cloudprovider.Instances - zones cloudprovider.Zones - server GRPCServer } // NodeInfo is information about a Kubernetes node. @@ -104,7 +99,7 @@ type NodeManager struct { connectionManager *cm.ConnectionManager // Reference to CPI-specific configuration - cpiCfg *CPIConfig + cfg *ccfg.CPIConfig // Mutexes nodeInfoLock sync.RWMutex diff --git a/pkg/cloudprovider/vsphere/vsphere_test.go b/pkg/cloudprovider/vsphere/vsphere_test.go index d9b9ba142..5e95cfe4c 100644 --- a/pkg/cloudprovider/vsphere/vsphere_test.go +++ b/pkg/cloudprovider/vsphere/vsphere_test.go @@ -20,7 +20,6 @@ import ( "context" "crypto/tls" "log" - "strings" "testing" lookup "github.com/vmware/govmomi/lookup/simulator" @@ -29,6 +28,7 @@ import ( sts "github.com/vmware/govmomi/sts/simulator" vapi "github.com/vmware/govmomi/vapi/simulator" + ccfg "k8s.io/cloud-provider-vsphere/pkg/cloudprovider/vsphere/config" vcfg "k8s.io/cloud-provider-vsphere/pkg/common/config" cm "k8s.io/cloud-provider-vsphere/pkg/common/connectionmanager" "k8s.io/cloud-provider-vsphere/pkg/common/vclib" @@ -132,22 +132,22 @@ func configFromSimWithTLS(tlsConfig *tls.Config, insecureAllowed bool, multiDc b } } -// configFromEnvOrSim returns config from configFromEnv if set, otherwise returns configFromSim. +// configFromEnvOrSim builds a config from configFromSim and overrides using configFromEnv func configFromEnvOrSim(multiDc bool) (*vcfg.Config, func()) { - cfg := &vcfg.Config{} + cfg, fin := configFromSim(multiDc) if err := cfg.FromEnv(); err != nil { - return configFromSim(multiDc) + return nil, nil } - return cfg, func() {} + return cfg, fin } func TestNewVSphere(t *testing.T) { - cfg := &CPIConfig{} + cfg := &ccfg.CPIConfig{} if err := cfg.FromCPIEnv(); err != nil { t.Skipf("No config found in environment") } - _, err := newVSphere(cfg) + _, err := newVSphere(cfg, nil) if err != nil { t.Fatalf("Failed to construct/authenticate vSphere: %s", err) } @@ -156,11 +156,11 @@ func TestNewVSphere(t *testing.T) { func TestVSphereLogin(t *testing.T) { initCfg, cleanup := configFromEnvOrSim(false) defer cleanup() - cfg := &CPIConfig{} + cfg := &ccfg.CPIConfig{} cfg.Config = *initCfg // Create vSphere configuration object - vs, err := newVSphere(cfg) + vs, err := newVSphere(cfg, nil) if err != nil { t.Fatalf("Failed to construct/authenticate vSphere: %s", err) } @@ -187,7 +187,7 @@ func TestVSphereLogin(t *testing.T) { func TestVSphereLoginByToken(t *testing.T) { initCfg, cleanup := configFromEnvOrSim(false) defer cleanup() - cfg := &CPIConfig{} + cfg := &ccfg.CPIConfig{} cfg.Config = *initCfg // Configure for SAML token auth @@ -195,7 +195,7 @@ func TestVSphereLoginByToken(t *testing.T) { cfg.Global.Password = localhostKey // Create vSphere configuration object - vs, err := newVSphere(cfg) + vs, err := newVSphere(cfg, nil) if err != nil { t.Fatalf("Failed to construct/authenticate vSphere: %s", err) } @@ -216,54 +216,6 @@ func TestVSphereLoginByToken(t *testing.T) { vcInstance.Conn.Logout(ctx) } -/* -func TestVSphereLoginWithCaCert(t *testing.T) { - caCertPEM, err := ioutil.ReadFile(fixtures.CaCertPath) - if err != nil { - t.Fatalf("Could not read ca cert from file") - } - - serverCert, err := tls.LoadX509KeyPair(fixtures.ServerCertPath, fixtures.ServerKeyPath) - if err != nil { - t.Fatalf("Could not load server cert and server key from files: %#v", err) - } - - certPool := x509.NewCertPool() - if ok := certPool.AppendCertsFromPEM(caCertPEM); !ok { - t.Fatalf("Cannot add CA to CAPool") - } - - tlsConfig := tls.Config{ - Certificates: []tls.Certificate{serverCert}, - RootCAs: certPool, - } - - cfg, cleanup := configFromSimWithTLS(&tlsConfig, false) - defer cleanup() - - cfg.Global.CAFile = fixtures.CaCertPath - - // Create vSphere configuration object - vs, err := newVSphere(cfg) - if err != nil { - t.Fatalf("Failed to construct/authenticate vSphere: %s", err) - } - - ctx := context.Background() - - // Create vSphere client - vcInstance, ok := vs.vsphereInstanceMap[cfg.Global.VCenterIP] - if !ok { - t.Fatalf("Couldn't get vSphere instance: %s", cfg.Global.VCenterIP) - } - - err = vcInstance.conn.Connect(ctx) - if err != nil { - t.Errorf("Failed to connect to vSphere: %s", err) - } - vcInstance.conn.Logout(ctx) -} -*/ func TestSecretVSphereConfig(t *testing.T) { var vs *VSphere var ( @@ -483,7 +435,7 @@ func TestSecretVSphereConfig(t *testing.T) { for _, testcase := range testcases { t.Logf("Executing Testcase: %s", testcase.testName) - cfg, err := ReadCPIConfig(strings.NewReader(testcase.conf)) + cfg, err := ccfg.ReadCPIConfig([]byte(testcase.conf)) if err != nil { if testcase.expectedError != nil { if err != testcase.expectedError { @@ -495,7 +447,7 @@ func TestSecretVSphereConfig(t *testing.T) { t.Fatalf("readConfig: unexpected error returned: %v", err) } } - vs, err = buildVSphereFromConfig(cfg) + vs, err = buildVSphereFromConfig(cfg, nil) if err != nil { // testcase.expectedError { t.Fatalf("buildVSphereFromConfig: Should succeed when a valid config is provided: %v", err) } diff --git a/pkg/common/config/config.go b/pkg/common/config/config.go index 8806be21b..2bcf19736 100644 --- a/pkg/common/config/config.go +++ b/pkg/common/config/config.go @@ -18,16 +18,18 @@ package config import ( "fmt" - "io" "os" "strconv" "strings" "k8s.io/klog" - - "gopkg.in/gcfg.v1" ) +/* + TODO: + When the INI based cloud-config is deprecated, this functions below should be preserved +*/ + func getEnvKeyValue(match string, partial bool) (string, string, error) { for _, e := range os.Environ() { pair := strings.Split(e, "=") @@ -143,13 +145,6 @@ func (cfg *Config) FromEnv() error { cfg.Labels.Zone = v } - if v := os.Getenv("VSPHERE_IP_FAMILY"); v != "" { - cfg.Global.IPFamily = v - } - if cfg.Global.IPFamily == "" { - cfg.Global.IPFamily = DefaultIPFamily - } - //Build VirtualCenter from ENVs for _, e := range os.Environ() { pair := strings.Split(e, "=") @@ -222,9 +217,10 @@ func (cfg *Config) FromEnv() error { secretRef = vcenter } + iPFamilyPriority := []string{DefaultIPFamily} _, ipFamily, errIPFamily := getEnvKeyValue("VCENTER_"+id+"_IP_FAMILY", false) if errIPFamily != nil { - ipFamily = cfg.Global.IPFamily + iPFamilyPriority = []string{ipFamily} } // If server is explicitly set, that means the vcenter value above is the TenantRef @@ -235,226 +231,68 @@ func (cfg *Config) FromEnv() error { tenantRef = vcenter } - cfg.VirtualCenter[tenantRef] = &VirtualCenterConfig{ - User: username, - Password: password, - TenantRef: tenantRef, - VCenterIP: vcenterIP, - VCenterPort: port, - InsecureFlag: insecureFlag, - Datacenters: datacenters, - RoundTripperCount: roundtrip, - CAFile: caFile, - Thumbprint: thumbprint, - SecretRef: secretRef, - SecretName: secretName, - SecretNamespace: secretNamespace, - IPFamily: ipFamily, + var vcc *VirtualCenterConfig + if cfg.VirtualCenter[tenantRef] != nil { + vcc = cfg.VirtualCenter[tenantRef] + } else { + vcc = &VirtualCenterConfig{} + cfg.VirtualCenter[tenantRef] = vcc } - } - } - if cfg.Global.VCenterIP != "" && cfg.VirtualCenter[cfg.Global.VCenterIP] == nil { - cfg.VirtualCenter[cfg.Global.VCenterIP] = &VirtualCenterConfig{ - User: cfg.Global.User, - Password: cfg.Global.Password, - TenantRef: cfg.Global.VCenterIP, - VCenterIP: cfg.Global.VCenterIP, - VCenterPort: cfg.Global.VCenterPort, - InsecureFlag: cfg.Global.InsecureFlag, - Datacenters: cfg.Global.Datacenters, - RoundTripperCount: cfg.Global.RoundTripperCount, - CAFile: cfg.Global.CAFile, - Thumbprint: cfg.Global.Thumbprint, - SecretRef: DefaultCredentialManager, - SecretName: cfg.Global.SecretName, - SecretNamespace: cfg.Global.SecretNamespace, - IPFamily: cfg.Global.IPFamily, + vcc.User = username + vcc.Password = password + vcc.TenantRef = tenantRef + vcc.VCenterIP = vcenterIP + vcc.VCenterPort = port + vcc.InsecureFlag = insecureFlag + vcc.Datacenters = datacenters + vcc.RoundTripperCount = roundtrip + vcc.CAFile = caFile + vcc.Thumbprint = thumbprint + vcc.SecretRef = secretRef + vcc.SecretName = secretName + vcc.SecretNamespace = secretNamespace + vcc.IPFamilyPriority = iPFamilyPriority } } - err := cfg.validateConfig() - if err != nil { - return err - } - return nil } -// IsSecretInfoProvided returns true if k8s secret is set or using generic CO secret method. -// If both k8s secret and generic CO both are true, we don't know which to use, so return false. -func (cfg *Config) IsSecretInfoProvided() bool { - return (cfg.Global.SecretName != "" && cfg.Global.SecretNamespace != "" && cfg.Global.SecretsDirectory == "") || - (cfg.Global.SecretName == "" && cfg.Global.SecretNamespace == "" && cfg.Global.SecretsDirectory != "") -} - -func validateIPFamily(value string) ([]string, error) { - if len(value) == 0 { - return []string{DefaultIPFamily}, nil - } - - ipFamilies := strings.Split(value, ",") - for i, ipFamily := range ipFamilies { - ipFamily = strings.TrimSpace(ipFamily) - if len(ipFamily) == 0 { - copy(ipFamilies[i:], ipFamilies[i+1:]) // Shift a[i+1:] left one index. - ipFamilies[len(ipFamilies)-1] = "" // Erase last element (write zero value). - ipFamilies = ipFamilies[:len(ipFamilies)-1] // Truncate slice. - continue - } - if !strings.EqualFold(ipFamily, IPv4Family) && !strings.EqualFold(ipFamily, IPv6Family) { - return nil, ErrInvalidIPFamilyType - } - } - - return ipFamilies, nil -} +/* + TODO: + When the INI based cloud-config is deprecated, the references to the + INI based code (ie the call to ReadConfigINI) below should be deleted. +*/ -func (cfg *Config) validateConfig() error { - //Fix default global values - if cfg.Global.RoundTripperCount == 0 { - cfg.Global.RoundTripperCount = DefaultRoundTripperCount - } - if cfg.Global.VCenterPort == "" { - cfg.Global.VCenterPort = DefaultVCenterPort - } - if cfg.Global.APIBinding == "" { - cfg.Global.APIBinding = DefaultAPIBinding - } - if cfg.Global.IPFamily == "" { - cfg.Global.IPFamily = DefaultIPFamily +// ReadConfig parses vSphere cloud config file and stores it into VSphereConfig. +// Environment variables are also checked +func ReadConfig(byConfig []byte) (*Config, error) { + if len(byConfig) == 0 { + return nil, fmt.Errorf("Invalid YAML/INI file") } - ipFamilyPriority, err := validateIPFamily(cfg.Global.IPFamily) + cfg, err := ReadConfigYAML(byConfig) if err != nil { - klog.Errorf("Invalid Global IPFamily: %s, err=%s", cfg.Global.IPFamily, err) - return err - } - - // Create a single instance of VSphereInstance for the Global VCenterIP if the - // VirtualCenter does not already exist in the map - if cfg.Global.VCenterIP != "" && cfg.VirtualCenter[cfg.Global.VCenterIP] == nil { - vcConfig := &VirtualCenterConfig{ - User: cfg.Global.User, - Password: cfg.Global.Password, - TenantRef: cfg.Global.VCenterIP, - VCenterIP: cfg.Global.VCenterIP, - VCenterPort: cfg.Global.VCenterPort, - InsecureFlag: cfg.Global.InsecureFlag, - Datacenters: cfg.Global.Datacenters, - RoundTripperCount: cfg.Global.RoundTripperCount, - CAFile: cfg.Global.CAFile, - Thumbprint: cfg.Global.Thumbprint, - SecretRef: DefaultCredentialManager, - SecretName: cfg.Global.SecretName, - SecretNamespace: cfg.Global.SecretNamespace, - IPFamily: cfg.Global.IPFamily, - IPFamilyPriority: ipFamilyPriority, - } - cfg.VirtualCenter[cfg.Global.VCenterIP] = vcConfig - } - - // Must have at least one vCenter defined - if len(cfg.VirtualCenter) == 0 { - klog.Error(ErrMissingVCenter) - return ErrMissingVCenter - } - - // vsphere.conf is no longer supported in the old format. - for vcServer, vcConfig := range cfg.VirtualCenter { - klog.V(4).Infof("Initializing vc server %s", vcServer) - if vcServer == "" { - klog.Error(ErrInvalidVCenterIP) - return ErrInvalidVCenterIP - } - - // If vcConfig.VCenterIP is explicitly set, that means the vcServer - // above is the TenantRef - if vcConfig.VCenterIP != "" { - //vcConfig.VCenterIP is already set - vcConfig.TenantRef = vcServer - } else { - vcConfig.VCenterIP = vcServer - vcConfig.TenantRef = vcServer - } - - if !cfg.IsSecretInfoProvided() && !vcConfig.IsSecretInfoProvided() { - if vcConfig.User == "" { - vcConfig.User = cfg.Global.User - if vcConfig.User == "" { - klog.Errorf("vcConfig.User is empty for vc %s!", vcServer) - return ErrUsernameMissing - } - } - if vcConfig.Password == "" { - vcConfig.Password = cfg.Global.Password - if vcConfig.Password == "" { - klog.Errorf("vcConfig.Password is empty for vc %s!", vcServer) - return ErrPasswordMissing - } - } - } else if cfg.IsSecretInfoProvided() && !vcConfig.IsSecretInfoProvided() { - vcConfig.SecretRef = DefaultCredentialManager - } else if vcConfig.IsSecretInfoProvided() { - vcConfig.SecretRef = vcConfig.SecretNamespace + "/" + vcConfig.SecretName - } - - if vcConfig.VCenterPort == "" { - vcConfig.VCenterPort = cfg.Global.VCenterPort - } - - if vcConfig.Datacenters == "" { - if cfg.Global.Datacenters != "" { - vcConfig.Datacenters = cfg.Global.Datacenters - } - } - if vcConfig.RoundTripperCount == 0 { - vcConfig.RoundTripperCount = cfg.Global.RoundTripperCount - } - if vcConfig.CAFile == "" { - vcConfig.CAFile = cfg.Global.CAFile - } - if vcConfig.Thumbprint == "" { - vcConfig.Thumbprint = cfg.Global.Thumbprint - } - - if vcConfig.IPFamily == "" { - vcConfig.IPFamily = cfg.Global.IPFamily - } + klog.Warningf("ReadConfigYAML failed: %s", err) - ipFamilyPriority, err := validateIPFamily(vcConfig.IPFamily) + cfg, err = ReadConfigINI(byConfig) if err != nil { - klog.Errorf("Invalid vcConfig IPFamily: %s, err=%s", vcConfig.IPFamily, err) - return err + klog.Errorf("ReadConfigINI failed: %s", err) + return nil, err } - vcConfig.IPFamilyPriority = ipFamilyPriority - insecure := vcConfig.InsecureFlag - if !insecure { - vcConfig.InsecureFlag = cfg.Global.InsecureFlag - } - } - - return nil -} - -// ReadConfig parses vSphere cloud config file and stores it into VSphereConfig. -// Environment variables are also checked -func ReadConfig(config io.Reader) (*Config, error) { - if config == nil { - return nil, fmt.Errorf("no vSphere cloud provider config file given") - } - - cfg := &Config{} - - if err := gcfg.FatalOnly(gcfg.ReadInto(cfg, config)); err != nil { - return nil, err + klog.Info("ReadConfig INI succeeded. INI-based cloud-config is deprecated and will be removed in 2.0. Please use YAML based cloud-config.") + } else { + klog.Info("ReadConfig YAML succeeded") } // Env Vars should override config file entries if present if err := cfg.FromEnv(); err != nil { + klog.Errorf("FromEnv failed: %s", err) return nil, err } + klog.Info("Config initialized") return cfg, nil } diff --git a/pkg/common/config/config_ini_legacy.go b/pkg/common/config/config_ini_legacy.go new file mode 100644 index 000000000..391ace6ef --- /dev/null +++ b/pkg/common/config/config_ini_legacy.go @@ -0,0 +1,267 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "fmt" + "strings" + + ini "gopkg.in/gcfg.v1" + "k8s.io/klog" +) + +/* + TODO: + When the INI based cloud-config is deprecated. This file should be deleted. +*/ + +// CreateConfig generates a common Config object based on what other structs and funcs +// are already dependent upon in other packages. +func (cci *CommonConfigINI) CreateConfig() *Config { + cfg := &Config{ + VirtualCenter: make(map[string]*VirtualCenterConfig), + } + + cfg.Global.User = cci.Global.User + cfg.Global.Password = cci.Global.Password + cfg.Global.VCenterIP = cci.Global.VCenterIP + cfg.Global.VCenterPort = cci.Global.VCenterPort + cfg.Global.InsecureFlag = cci.Global.InsecureFlag + cfg.Global.Datacenters = cci.Global.Datacenters + cfg.Global.RoundTripperCount = cci.Global.RoundTripperCount + cfg.Global.CAFile = cci.Global.CAFile + cfg.Global.Thumbprint = cci.Global.Thumbprint + cfg.Global.SecretName = cci.Global.SecretName + cfg.Global.SecretNamespace = cci.Global.SecretNamespace + cfg.Global.SecretsDirectory = cci.Global.SecretsDirectory + cfg.Global.APIDisable = cci.Global.APIDisable + cfg.Global.APIBinding = cci.Global.APIBinding + + for keyVcConfig, valVcConfig := range cci.VirtualCenter { + cfg.VirtualCenter[keyVcConfig] = &VirtualCenterConfig{ + User: valVcConfig.User, + Password: valVcConfig.Password, + TenantRef: valVcConfig.TenantRef, + VCenterIP: valVcConfig.VCenterIP, + VCenterPort: valVcConfig.VCenterPort, + InsecureFlag: valVcConfig.InsecureFlag, + Datacenters: valVcConfig.Datacenters, + RoundTripperCount: valVcConfig.RoundTripperCount, + CAFile: valVcConfig.CAFile, + Thumbprint: valVcConfig.Thumbprint, + SecretRef: valVcConfig.SecretRef, + SecretName: valVcConfig.SecretName, + SecretNamespace: valVcConfig.SecretNamespace, + IPFamilyPriority: valVcConfig.IPFamilyPriority, + } + } + + cfg.Labels.Region = cci.Labels.Region + cfg.Labels.Zone = cci.Labels.Zone + + return cfg +} + +// validateIPFamily takes the possible values of IPFamily and initializes the +// slice as determined bby priority +func (vcci *VirtualCenterConfigINI) validateIPFamily() error { + if len(vcci.IPFamily) == 0 { + vcci.IPFamily = DefaultIPFamily + } + + ipFamilies := strings.Split(vcci.IPFamily, ",") + for i, ipFamily := range ipFamilies { + ipFamily = strings.TrimSpace(ipFamily) + if len(ipFamily) == 0 { + copy(ipFamilies[i:], ipFamilies[i+1:]) // Shift a[i+1:] left one index. + ipFamilies[len(ipFamilies)-1] = "" // Erase last element (write zero value). + ipFamilies = ipFamilies[:len(ipFamilies)-1] // Truncate slice. + continue + } + if !strings.EqualFold(ipFamily, IPv4Family) && !strings.EqualFold(ipFamily, IPv6Family) { + return ErrInvalidIPFamilyType + } + } + + vcci.IPFamilyPriority = ipFamilies + return nil +} + +// isSecretInfoProvided returns true if k8s secret is set or using generic CO secret method. +// If both k8s secret and generic CO both are true, we don't know which to use, so return false. +func (cci *CommonConfigINI) isSecretInfoProvided() bool { + return (cci.Global.SecretName != "" && cci.Global.SecretNamespace != "" && cci.Global.SecretsDirectory == "") || + (cci.Global.SecretName == "" && cci.Global.SecretNamespace == "" && cci.Global.SecretsDirectory != "") +} + +// isSecretInfoProvided returns true if the secret per VC has been configured +func (vcci *VirtualCenterConfigINI) isSecretInfoProvided() bool { + return vcci.SecretName != "" && vcci.SecretNamespace != "" +} + +func (cci *CommonConfigINI) validateConfig() error { + //Fix default global values + if cci.Global.RoundTripperCount == 0 { + cci.Global.RoundTripperCount = DefaultRoundTripperCount + } + if cci.Global.VCenterPort == "" { + cci.Global.VCenterPort = DefaultVCenterPortStr + } + if cci.Global.APIBinding == "" { + cci.Global.APIBinding = DefaultAPIBinding + } + if cci.Global.IPFamily == "" { + cci.Global.IPFamily = DefaultIPFamily + } + + // Create a single instance of VSphereInstance for the Global VCenterIP if the + // VirtualCenter does not already exist in the map + if cci.Global.VCenterIP != "" && cci.VirtualCenter[cci.Global.VCenterIP] == nil { + cci.VirtualCenter[cci.Global.VCenterIP] = &VirtualCenterConfigINI{ + User: cci.Global.User, + Password: cci.Global.Password, + TenantRef: cci.Global.VCenterIP, + VCenterIP: cci.Global.VCenterIP, + VCenterPort: cci.Global.VCenterPort, + InsecureFlag: cci.Global.InsecureFlag, + Datacenters: cci.Global.Datacenters, + RoundTripperCount: cci.Global.RoundTripperCount, + CAFile: cci.Global.CAFile, + Thumbprint: cci.Global.Thumbprint, + SecretRef: DefaultCredentialManager, + SecretName: cci.Global.SecretName, + SecretNamespace: cci.Global.SecretNamespace, + IPFamily: cci.Global.IPFamily, + } + } + + // Must have at least one vCenter defined + if len(cci.VirtualCenter) == 0 { + klog.Error(ErrMissingVCenter) + return ErrMissingVCenter + } + + // vsphere.conf is no longer supported in the old format. + for vcServer, vcConfig := range cci.VirtualCenter { + klog.V(4).Infof("Initializing vc server %s", vcServer) + if vcServer == "" { + klog.Error(ErrInvalidVCenterIP) + return ErrInvalidVCenterIP + } + + // If vcConfig.VCenterIP is explicitly set, that means the vcServer + // above is the TenantRef + if vcConfig.VCenterIP != "" { + //vcConfig.VCenterIP is already set + vcConfig.TenantRef = vcServer + } else { + vcConfig.VCenterIP = vcServer + vcConfig.TenantRef = vcServer + } + + if !cci.isSecretInfoProvided() && !vcConfig.isSecretInfoProvided() { + if vcConfig.User == "" { + vcConfig.User = cci.Global.User + if vcConfig.User == "" { + klog.Errorf("vcConfig.User is empty for vc %s!", vcServer) + return ErrUsernameMissing + } + } + if vcConfig.Password == "" { + vcConfig.Password = cci.Global.Password + if vcConfig.Password == "" { + klog.Errorf("vcConfig.Password is empty for vc %s!", vcServer) + return ErrPasswordMissing + } + } + } else if cci.isSecretInfoProvided() && !vcConfig.isSecretInfoProvided() { + vcConfig.SecretRef = DefaultCredentialManager + } else if vcConfig.isSecretInfoProvided() { + vcConfig.SecretRef = vcConfig.SecretNamespace + "/" + vcConfig.SecretName + } + + if vcConfig.VCenterPort == "" { + vcConfig.VCenterPort = cci.Global.VCenterPort + } + + if vcConfig.Datacenters == "" { + if cci.Global.Datacenters != "" { + vcConfig.Datacenters = cci.Global.Datacenters + } + } + if vcConfig.RoundTripperCount == 0 { + vcConfig.RoundTripperCount = cci.Global.RoundTripperCount + } + if vcConfig.CAFile == "" { + vcConfig.CAFile = cci.Global.CAFile + } + if vcConfig.Thumbprint == "" { + vcConfig.Thumbprint = cci.Global.Thumbprint + } + + if vcConfig.IPFamily == "" { + vcConfig.IPFamily = cci.Global.IPFamily + } + + err := vcConfig.validateIPFamily() + if err != nil { + klog.Errorf("Invalid vcConfig IPFamily: %s, err=%s", vcConfig.IPFamily, err) + return err + } + + insecure := vcConfig.InsecureFlag + if !insecure { + vcConfig.InsecureFlag = cci.Global.InsecureFlag + } + } + + return nil +} + +// ReadRawConfigINI parses vSphere cloud config file and stores it into ConfigINI +func ReadRawConfigINI(byConfig []byte) (*CommonConfigINI, error) { + if len(byConfig) == 0 { + return nil, fmt.Errorf("Invalid INI file") + } + + strConfig := string(byConfig[:]) + + cfg := &CommonConfigINI{ + VirtualCenter: make(map[string]*VirtualCenterConfigINI), + } + + if err := ini.FatalOnly(ini.ReadStringInto(cfg, strConfig)); err != nil { + return nil, err + } + + err := cfg.validateConfig() + if err != nil { + return nil, err + } + + return cfg, nil +} + +// ReadConfigINI parses vSphere cloud config file and stores it into Config +func ReadConfigINI(byConfig []byte) (*Config, error) { + cfg, err := ReadRawConfigINI(byConfig) + if err != nil { + return nil, err + } + + return cfg.CreateConfig(), nil +} diff --git a/pkg/common/config/config_test.go b/pkg/common/config/config_ini_legacy_test.go similarity index 79% rename from pkg/common/config/config_test.go rename to pkg/common/config/config_ini_legacy_test.go index c2f78cbd8..7b964997e 100644 --- a/pkg/common/config/config_test.go +++ b/pkg/common/config/config_ini_legacy_test.go @@ -17,12 +17,16 @@ limitations under the License. package config import ( - "os" "strings" "testing" ) -const basicConfig = ` +/* + TODO: + When the INI based cloud-config is deprecated. This file should be deleted. +*/ + +const basicConfigINI = ` [Global] server = 0.0.0.0 port = 443 @@ -33,7 +37,7 @@ datacenters = us-west ca-file = /some/path/to/a/ca.pem ` -const multiVCDCsUsingSecretConfig = ` +const multiVCDCsUsingSecretConfigINI = ` [Global] port = 443 insecure-flag = true @@ -59,13 +63,13 @@ secret-namespace = "kube-system" # port, insecure-flag will be used from Global section. ` -func TestReadConfigGlobal(t *testing.T) { - _, err := ReadConfig(nil) +func TestReadConfigINIGlobal(t *testing.T) { + _, err := ReadConfigINI([]byte("")) if err == nil { t.Errorf("Should fail when no config is provided: %s", err) } - cfg, err := ReadConfig(strings.NewReader(basicConfig)) + cfg, err := ReadConfigINI([]byte(basicConfigINI)) if err != nil { t.Fatalf("Should succeed when a valid config is provided: %s", err) } @@ -83,12 +87,23 @@ func TestReadConfigGlobal(t *testing.T) { } } +/* +TODO: move to global +func TestBlankEnvFails(t *testing.T) { + cfg := &ConfigINI{} + + err := cfg.FromEnv() + if err == nil { + t.Fatalf("Env only config should fail if env not set") + } +} + func TestEnvOverridesFile(t *testing.T) { ip := "127.0.0.1" os.Setenv("VSPHERE_VCENTER", ip) defer os.Unsetenv("VSPHERE_VCENTER") - cfg, err := ReadConfig(strings.NewReader(basicConfig)) + cfg, err := ReadConfigINI([]byte(basicConfigINI)) if err != nil { t.Fatalf("Should succeed when a valid config is provided: %s", err) } @@ -97,87 +112,74 @@ func TestEnvOverridesFile(t *testing.T) { t.Errorf("expected IP: %s, got: %s", ip, cfg.Global.VCenterIP) } } +*/ -func TestBlankEnvFails(t *testing.T) { - cfg := &Config{} - - err := cfg.FromEnv() - if err == nil { - t.Fatalf("Env only config should fail if env not set") - } -} +func TestIPFamiliesINI(t *testing.T) { + vcci := VirtualCenterConfigINI{} -func TestIPFamilies(t *testing.T) { - input := "ipv6" - ipFamilies, err := validateIPFamily(input) + vcci.IPFamily = "ipv6" + err := vcci.validateIPFamily() if err != nil { t.Errorf("Valid ipv6 but yielded err: %s", err) } - size := len(ipFamilies) + size := len(vcci.IPFamilyPriority) if size != 1 { t.Errorf("Invalid family list expected: 1, actual: %d", size) } - input = "ipv4" - ipFamilies, err = validateIPFamily(input) + vcci.IPFamily = "ipv4" + err = vcci.validateIPFamily() if err != nil { t.Errorf("Valid ipv4 but yielded err: %s", err) } - size = len(ipFamilies) + size = len(vcci.IPFamilyPriority) if size != 1 { t.Errorf("Invalid family list expected: 1, actual: %d", size) } - input = "ipv4, " - ipFamilies, err = validateIPFamily(input) + vcci.IPFamily = "ipv4, " + err = vcci.validateIPFamily() if err != nil { t.Errorf("Valid ipv4, but yielded err: %s", err) } - size = len(ipFamilies) + size = len(vcci.IPFamilyPriority) if size != 1 { t.Errorf("Invalid family list expected: 1, actual: %d", size) } - input = "ipv6,ipv4" - ipFamilies, err = validateIPFamily(input) + vcci.IPFamily = "ipv6,ipv4" + err = vcci.validateIPFamily() if err != nil { t.Errorf("Valid ipv6/ipv4 but yielded err: %s", err) } - size = len(ipFamilies) + size = len(vcci.IPFamilyPriority) if size != 2 { t.Errorf("Invalid family list expected: 2, actual: %d", size) } - input = "ipv7" - _, err = validateIPFamily(input) + vcci.IPFamily = "ipv7" + err = vcci.validateIPFamily() if err == nil { t.Errorf("Invalid ipv7 but successful") } - input = "ipv4,ipv7" - _, err = validateIPFamily(input) + vcci.IPFamily = "ipv4,ipv7" + err = vcci.validateIPFamily() if err == nil { t.Errorf("Invalid ipv4,ipv7 but successful") } } -func TestTenantRefs(t *testing.T) { - cfg, err := ReadConfig(strings.NewReader(multiVCDCsUsingSecretConfig)) +func TestTenantRefsINI(t *testing.T) { + cfg, err := ReadConfigINI([]byte(multiVCDCsUsingSecretConfigINI)) if err != nil { t.Fatalf("Should succeed when a valid config is provided: %s", err) } - if cfg.IsSecretInfoProvided() { - t.Error("IsSecretInfoProvided should not be set.") - } - vcConfig1 := cfg.VirtualCenter["tenant1"] if vcConfig1 == nil { t.Fatalf("Should return a valid vcConfig1") } - if !vcConfig1.IsSecretInfoProvided() { - t.Error("vcConfig1.IsSecretInfoProvided() should be set.") - } if !strings.EqualFold(vcConfig1.VCenterIP, "10.0.0.1") { t.Errorf("vcConfig1 VCenterIP should be 10.0.0.1 but actual=%s", vcConfig1.VCenterIP) } @@ -192,9 +194,6 @@ func TestTenantRefs(t *testing.T) { if vcConfig2 == nil { t.Fatalf("Should return a valid vcConfig2") } - if !vcConfig2.IsSecretInfoProvided() { - t.Error("vcConfig2.IsSecretInfoProvided() should be set.") - } if !strings.EqualFold(vcConfig2.VCenterIP, "10.0.0.2") { t.Errorf("vcConfig2 VCenterIP should be 10.0.0.2 but actual=%s", vcConfig2.VCenterIP) } @@ -209,9 +208,6 @@ func TestTenantRefs(t *testing.T) { if vcConfig3 == nil { t.Fatalf("Should return a valid vcConfig3") } - if !vcConfig3.IsSecretInfoProvided() { - t.Error("vcConfig3.IsSecretInfoProvided() should be set.") - } if !strings.EqualFold(vcConfig3.VCenterIP, "10.0.0.3") { t.Errorf("vcConfig3 VCenterIP should be 10.0.0.3 but actual=%s", vcConfig3.VCenterIP) } diff --git a/pkg/common/config/config_yaml.go b/pkg/common/config/config_yaml.go new file mode 100644 index 000000000..67a9921b2 --- /dev/null +++ b/pkg/common/config/config_yaml.go @@ -0,0 +1,231 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "fmt" + "strings" + + yaml "gopkg.in/yaml.v2" + "k8s.io/klog" +) + +/* + TODO: + When the INI based cloud-config is deprecated, this file should be merged into config.go + and this file should be deleted. +*/ + +// CreateConfig generates a common Config object based on what other structs and funcs +// are already dependent upon in other packages. +func (ccy *CommonConfigYAML) CreateConfig() *Config { + cfg := &Config{ + VirtualCenter: make(map[string]*VirtualCenterConfig), + } + + cfg.Global.User = ccy.Global.User + cfg.Global.Password = ccy.Global.Password + cfg.Global.VCenterIP = ccy.Global.VCenterIP + cfg.Global.VCenterPort = fmt.Sprint(ccy.Global.VCenterPort) + cfg.Global.InsecureFlag = ccy.Global.InsecureFlag + cfg.Global.Datacenters = strings.Join(ccy.Global.Datacenters, ",") + cfg.Global.RoundTripperCount = ccy.Global.RoundTripperCount + cfg.Global.CAFile = ccy.Global.CAFile + cfg.Global.Thumbprint = ccy.Global.Thumbprint + cfg.Global.SecretName = ccy.Global.SecretName + cfg.Global.SecretNamespace = ccy.Global.SecretNamespace + cfg.Global.SecretsDirectory = ccy.Global.SecretsDirectory + cfg.Global.APIDisable = ccy.Global.APIDisable + cfg.Global.APIBinding = ccy.Global.APIBinding + + for keyVcConfig, valVcConfig := range ccy.Vcenter { + cfg.VirtualCenter[keyVcConfig] = &VirtualCenterConfig{ + User: valVcConfig.User, + Password: valVcConfig.Password, + TenantRef: valVcConfig.TenantRef, + VCenterIP: valVcConfig.VCenterIP, + VCenterPort: fmt.Sprint(valVcConfig.VCenterPort), + InsecureFlag: valVcConfig.InsecureFlag, + Datacenters: strings.Join(valVcConfig.Datacenters, ","), + RoundTripperCount: valVcConfig.RoundTripperCount, + CAFile: valVcConfig.CAFile, + Thumbprint: valVcConfig.Thumbprint, + SecretRef: valVcConfig.SecretRef, + SecretName: valVcConfig.SecretName, + SecretNamespace: valVcConfig.SecretNamespace, + IPFamilyPriority: valVcConfig.IPFamilyPriority, + } + } + + cfg.Labels.Region = ccy.Labels.Region + cfg.Labels.Zone = ccy.Labels.Zone + + return cfg +} + +// isSecretInfoProvided returns true if k8s secret is set or using generic CO secret method. +// If both k8s secret and generic CO both are true, we don't know which to use, so return false. +func (ccy *CommonConfigYAML) isSecretInfoProvided() bool { + return (ccy.Global.SecretName != "" && ccy.Global.SecretNamespace != "" && ccy.Global.SecretsDirectory == "") || + (ccy.Global.SecretName == "" && ccy.Global.SecretNamespace == "" && ccy.Global.SecretsDirectory != "") +} + +// isSecretInfoProvided returns true if the secret per VC has been configured +func (vccy *VirtualCenterConfigYAML) isSecretInfoProvided() bool { + return vccy.SecretName != "" && vccy.SecretNamespace != "" +} + +func (ccy *CommonConfigYAML) validateConfig() error { + //Fix default global values + if ccy.Global.RoundTripperCount == 0 { + ccy.Global.RoundTripperCount = DefaultRoundTripperCount + } + if ccy.Global.VCenterPort == 0 { + ccy.Global.VCenterPort = DefaultVCenterPort + } + if ccy.Global.APIBinding == "" { + ccy.Global.APIBinding = DefaultAPIBinding + } + if len(ccy.Global.IPFamilyPriority) == 0 { + ccy.Global.IPFamilyPriority = []string{DefaultIPFamily} + } + + // Create a single instance of VSphereInstance for the Global VCenterIP if the + // VirtualCenter does not already exist in the map + if ccy.Global.VCenterIP != "" && ccy.Vcenter[ccy.Global.VCenterIP] == nil { + ccy.Vcenter[ccy.Global.VCenterIP] = &VirtualCenterConfigYAML{ + User: ccy.Global.User, + Password: ccy.Global.Password, + TenantRef: ccy.Global.VCenterIP, + VCenterIP: ccy.Global.VCenterIP, + VCenterPort: ccy.Global.VCenterPort, + InsecureFlag: ccy.Global.InsecureFlag, + Datacenters: ccy.Global.Datacenters, + RoundTripperCount: ccy.Global.RoundTripperCount, + CAFile: ccy.Global.CAFile, + Thumbprint: ccy.Global.Thumbprint, + SecretRef: DefaultCredentialManager, + SecretName: ccy.Global.SecretName, + SecretNamespace: ccy.Global.SecretNamespace, + IPFamilyPriority: ccy.Global.IPFamilyPriority, + } + } + + // Must have at least one vCenter defined + if len(ccy.Vcenter) == 0 { + klog.Error(ErrMissingVCenter) + return ErrMissingVCenter + } + + // vsphere.conf is no longer supported in the old format. + for tenantRef, vcConfig := range ccy.Vcenter { + klog.V(4).Infof("Initializing vc server %s", tenantRef) + if vcConfig.VCenterIP == "" { + klog.Error(ErrInvalidVCenterIP) + return ErrInvalidVCenterIP + } + + // in the YAML-based config, the tenant ref is required in the config + vcConfig.TenantRef = tenantRef + + if !ccy.isSecretInfoProvided() && !vcConfig.isSecretInfoProvided() { + if vcConfig.User == "" { + vcConfig.User = ccy.Global.User + if vcConfig.User == "" { + klog.Errorf("vcConfig.User is empty for vc %s!", tenantRef) + return ErrUsernameMissing + } + } + if vcConfig.Password == "" { + vcConfig.Password = ccy.Global.Password + if vcConfig.Password == "" { + klog.Errorf("vcConfig.Password is empty for vc %s!", tenantRef) + return ErrPasswordMissing + } + } + } else if ccy.isSecretInfoProvided() && !vcConfig.isSecretInfoProvided() { + vcConfig.SecretRef = DefaultCredentialManager + } else if vcConfig.isSecretInfoProvided() { + vcConfig.SecretRef = vcConfig.SecretNamespace + "/" + vcConfig.SecretName + } + + if vcConfig.VCenterPort == 0 { + vcConfig.VCenterPort = ccy.Global.VCenterPort + } + + if len(vcConfig.Datacenters) == 0 { + if len(ccy.Global.Datacenters) != 0 { + vcConfig.Datacenters = ccy.Global.Datacenters + } + } + if vcConfig.RoundTripperCount == 0 { + vcConfig.RoundTripperCount = ccy.Global.RoundTripperCount + } + if vcConfig.CAFile == "" { + vcConfig.CAFile = ccy.Global.CAFile + } + if vcConfig.Thumbprint == "" { + vcConfig.Thumbprint = ccy.Global.Thumbprint + } + + if len(vcConfig.IPFamilyPriority) == 0 { + vcConfig.IPFamilyPriority = ccy.Global.IPFamilyPriority + } + + insecure := vcConfig.InsecureFlag + if !insecure { + vcConfig.InsecureFlag = ccy.Global.InsecureFlag + } + } + + return nil +} + +// ReadRawConfigYAML parses vSphere cloud config file and stores it into ConfigYAML +func ReadRawConfigYAML(byConfig []byte) (*CommonConfigYAML, error) { + if len(byConfig) == 0 { + klog.Errorf("Invalid YAML file") + return nil, fmt.Errorf("Invalid YAML file") + } + + cfg := CommonConfigYAML{ + Vcenter: make(map[string]*VirtualCenterConfigYAML), + } + + if err := yaml.Unmarshal(byConfig, &cfg); err != nil { + klog.Errorf("Unmarshal failed: %s", err) + return nil, err + } + + err := cfg.validateConfig() + if err != nil { + klog.Errorf("validateConfig failed: %s", err) + return nil, err + } + + return &cfg, nil +} + +// ReadConfigYAML parses vSphere cloud config file and stores it into Config +func ReadConfigYAML(byConfig []byte) (*Config, error) { + cfg, err := ReadRawConfigYAML(byConfig) + if err != nil { + return nil, err + } + + return cfg.CreateConfig(), nil +} diff --git a/pkg/common/config/config_yaml_test.go b/pkg/common/config/config_yaml_test.go new file mode 100644 index 000000000..5d3ce799e --- /dev/null +++ b/pkg/common/config/config_yaml_test.go @@ -0,0 +1,139 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "strings" + "testing" +) + +/* + TODO: + When the INI based cloud-config is deprecated, this file should be merged into config_test.go + and this file should be deleted. +*/ + +const basicConfigYAML = ` +global: + server: 0.0.0.0 + port: 443 + user: user + password: password + insecureFlag: true + datacenters: + - us-west + caFile: /some/path/to/a/ca.pem +` + +const multiVCDCsUsingSecretConfigYAML = ` +global: + port: 443 + insecureFlag: true + +vcenter: + tenant1: + server: 10.0.0.1 + datacenters: + - vic0dc + secretName: tenant1-secret + secretNamespace: kube-system + tenant2: + server: 10.0.0.2 + datacenters: + - vic1dc + secretName: tenant2-secret + secretNamespace: kube-system + 10.0.0.3: + server: 10.0.0.3 + datacenters: + - vicdc + secretName: eu-secret + secretNamespace: kube-system +` + +func TestReadConfigYAMLGlobal(t *testing.T) { + _, err := ReadConfigYAML([]byte("")) + if err == nil { + t.Errorf("Should fail when no config is provided: %s", err) + } + + cfg, err := ReadConfigYAML([]byte(basicConfigYAML)) + if err != nil { + t.Fatalf("Should succeed when a valid config is provided: %s", err) + } + + if cfg.Global.VCenterIP != "0.0.0.0" { + t.Errorf("incorrect vcenter ip: %s", cfg.Global.VCenterIP) + } + + if cfg.Global.Datacenters != "us-west" { + t.Errorf("incorrect datacenter: %s", cfg.Global.Datacenters) + } + + if cfg.Global.CAFile != "/some/path/to/a/ca.pem" { + t.Errorf("incorrect caFile: %s", cfg.Global.CAFile) + } +} + +func TestTenantRefsYAML(t *testing.T) { + cfg, err := ReadConfigYAML([]byte(multiVCDCsUsingSecretConfigYAML)) + if err != nil { + t.Fatalf("Should succeed when a valid config is provided: %s", err) + } + + vcConfig1 := cfg.VirtualCenter["tenant1"] + if vcConfig1 == nil { + t.Fatalf("Should return a valid vcConfig1") + } + if !strings.EqualFold(vcConfig1.VCenterIP, "10.0.0.1") { + t.Errorf("vcConfig1 VCenterIP should be 10.0.0.1 but actual=%s", vcConfig1.VCenterIP) + } + if !strings.EqualFold(vcConfig1.TenantRef, "tenant1") { + t.Errorf("vcConfig1 TenantRef should be tenant1 but actual=%s", vcConfig1.TenantRef) + } + if !strings.EqualFold(vcConfig1.SecretRef, "kube-system/tenant1-secret") { + t.Errorf("vcConfig1 SecretRef should be kube-system/tenant1-secret but actual=%s", vcConfig1.SecretRef) + } + + vcConfig2 := cfg.VirtualCenter["tenant2"] + if vcConfig2 == nil { + t.Fatalf("Should return a valid vcConfig2") + } + if !strings.EqualFold(vcConfig2.VCenterIP, "10.0.0.2") { + t.Errorf("vcConfig2 VCenterIP should be 10.0.0.2 but actual=%s", vcConfig2.VCenterIP) + } + if !strings.EqualFold(vcConfig2.TenantRef, "tenant2") { + t.Errorf("vcConfig2 TenantRef should be tenant2 but actual=%s", vcConfig2.TenantRef) + } + if !strings.EqualFold(vcConfig2.SecretRef, "kube-system/tenant2-secret") { + t.Errorf("vcConfig2 SecretRef should be kube-system/tenant2-secret but actual=%s", vcConfig2.SecretRef) + } + + vcConfig3 := cfg.VirtualCenter["10.0.0.3"] + if vcConfig3 == nil { + t.Fatalf("Should return a valid vcConfig3") + } + if !strings.EqualFold(vcConfig3.VCenterIP, "10.0.0.3") { + t.Errorf("vcConfig3 VCenterIP should be 10.0.0.3 but actual=%s", vcConfig3.VCenterIP) + } + if !strings.EqualFold(vcConfig3.TenantRef, "10.0.0.3") { + t.Errorf("vcConfig3 TenantRef should be eu-secret but actual=%s", vcConfig3.TenantRef) + } + if !strings.EqualFold(vcConfig3.SecretRef, "kube-system/eu-secret") { + t.Errorf("vcConfig3 SecretRef should be kube-system/eu-secret but actual=%s", vcConfig3.SecretRef) + } +} diff --git a/pkg/common/config/consts_and_errors.go b/pkg/common/config/consts_and_errors.go index d65895e80..0ed200ffb 100644 --- a/pkg/common/config/consts_and_errors.go +++ b/pkg/common/config/consts_and_errors.go @@ -29,8 +29,10 @@ const ( // exposing the API service. DefaultAPIBinding string = ":43001" - // DefaultVCenterPort is the default port used to access vCenter. - DefaultVCenterPort string = "443" + // DefaultVCenterPortStr is the default port used to access vCenter in string form + DefaultVCenterPortStr string = "443" + // DefaultVCenterPort is the default port used to access vCenter in uint form + DefaultVCenterPort uint = 443 // DefaultSecretDirectory is the default path to the secrets directory. DefaultSecretDirectory string = "/etc/cloud/secrets" diff --git a/pkg/common/config/types_common.go b/pkg/common/config/types_common.go new file mode 100644 index 000000000..f7bd32d92 --- /dev/null +++ b/pkg/common/config/types_common.go @@ -0,0 +1,123 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +/* + TODO: + When the INI based cloud-config is deprecated. This file should be deleted and + the structs in types_yaml.go will be renamed to replace the ones in this file. +*/ + +// Global struct +type Global struct { + // vCenter username. + User string + // vCenter password in clear text. + Password string + // Deprecated. Use VirtualCenter to specify multiple vCenter Servers. + // vCenter IP. + VCenterIP string + // vCenter port. + VCenterPort string + // True if vCenter uses self-signed cert. + InsecureFlag bool + // Datacenter in which VMs are located. + Datacenters string + // Soap round tripper count (retries = RoundTripper - 1) + RoundTripperCount uint + // Specifies the path to a CA certificate in PEM format. Optional; if not + // configured, the system's CA certificates will be used. + CAFile string + // Thumbprint of the VCenter's certificate thumbprint + Thumbprint string + // Name of the secret were vCenter credentials are present. + SecretName string + // Secret Namespace where secret will be present that has vCenter credentials. + SecretNamespace string + // Secret directory in the event that: + // 1) we don't want to use the k8s API to listen for changes to secrets + // 2) we are not in a k8s env, namely DC/OS, since CSI is CO agnostic + // Default: /etc/cloud/credentials + SecretsDirectory string + // Disable the vSphere CCM API + // Default: true + APIDisable bool + // Configurable vSphere CCM API port + // Default: 43001 + APIBinding string +} + +// VirtualCenterConfig struct +type VirtualCenterConfig struct { + // vCenter username. + User string + // vCenter password in clear text. + Password string + // TenantRef (intentionally not exposed via the config) is a unique tenant ref to + // be used in place of the vcServer as the primary connection key. If one label is set, + // all virtual center configs must have a unique label. + TenantRef string + // vCenterIP - If this field in the config is set, it is assumed then that value in [VirtualCenter ""] + // is now the TenantRef above and this field is the actual VCenterIP. Otherwise for backward + // compatibility, the value by default is the IP or FQDN of the vCenter Server. + VCenterIP string + // vCenter port. + VCenterPort string + // True if vCenter uses self-signed cert. + InsecureFlag bool + // Datacenter in which VMs are located. + Datacenters string + // Soap round tripper count (retries = RoundTripper - 1) + RoundTripperCount uint + // Specifies the path to a CA certificate in PEM format. Optional; if not + // configured, the system's CA certificates will be used. + CAFile string + // Thumbprint of the VCenter's certificate thumbprint + Thumbprint string + // SecretRef (intentionally not exposed via the config) is a key to identify which + // InformerManager holds the secret + SecretRef string + // Name of the secret where vCenter credentials are present. + SecretName string + // Namespace where the secret will be present containing vCenter credentials. + SecretNamespace string + // IP Family enables the ability to support IPv4 or IPv6 + // Supported values are: + // ipv4 - IPv4 addresses only (Default) + // ipv6 - IPv6 addresses only + IPFamilyPriority []string +} + +// Labels struct +type Labels struct { + // Zone describes a zone + Zone string + // Region describes a region + Region string +} + +// Config is used to read and store information from the cloud configuration file +type Config struct { + // Global settings + Global Global + + // Virtual Center configurations + VirtualCenter map[string]*VirtualCenterConfig + + // Tag categories and tags which correspond to "built-in node labels: zones and region" + Labels Labels +} diff --git a/pkg/common/config/types.go b/pkg/common/config/types_ini_legacy.go similarity index 52% rename from pkg/common/config/types.go rename to pkg/common/config/types_ini_legacy.go index fca50dd80..d31eae215 100644 --- a/pkg/common/config/types.go +++ b/pkg/common/config/types_ini_legacy.go @@ -16,64 +16,58 @@ limitations under the License. package config -// Config is used to read and store information from the cloud configuration file -type Config struct { - Global struct { - // vCenter username. - User string `gcfg:"user"` - // vCenter password in clear text. - Password string `gcfg:"password"` - // Deprecated. Use VirtualCenter to specify multiple vCenter Servers. - // vCenter IP. - VCenterIP string `gcfg:"server"` - // vCenter port. - VCenterPort string `gcfg:"port"` - // True if vCenter uses self-signed cert. - InsecureFlag bool `gcfg:"insecure-flag"` - // Datacenter in which VMs are located. - Datacenters string `gcfg:"datacenters"` - // Soap round tripper count (retries = RoundTripper - 1) - RoundTripperCount uint `gcfg:"soap-roundtrip-count"` - // Specifies the path to a CA certificate in PEM format. Optional; if not - // configured, the system's CA certificates will be used. - CAFile string `gcfg:"ca-file"` - // Thumbprint of the VCenter's certificate thumbprint - Thumbprint string `gcfg:"thumbprint"` - // Name of the secret were vCenter credentials are present. - SecretName string `gcfg:"secret-name"` - // Secret Namespace where secret will be present that has vCenter credentials. - SecretNamespace string `gcfg:"secret-namespace"` - // Secret directory in the event that: - // 1) we don't want to use the k8s API to listen for changes to secrets - // 2) we are not in a k8s env, namely DC/OS, since CSI is CO agnostic - // Default: /etc/cloud/credentials - SecretsDirectory string `gcfg:"secrets-directory"` - // Disable the vSphere CCM API - // Default: true - APIDisable bool `gcfg:"api-disable"` - // Configurable vSphere CCM API port - // Default: 43001 - APIBinding string `gcfg:"api-binding"` - // IP Family enables the ability to support IPv4 or IPv6 - // Supported values are: - // ipv4 - IPv4 addresses only (Default) - // ipv6 - IPv6 addresses only - IPFamily string `gcfg:"ip-family"` - } - - // Virtual Center configurations - VirtualCenter map[string]*VirtualCenterConfig +/* + TODO: + When the INI based cloud-config is deprecated. This file should be deleted. +*/ - // Tag categories and tags which correspond to "built-in node labels: zones and region" - Labels struct { - Zone string `gcfg:"zone"` - Region string `gcfg:"region"` - } +// GlobalINI are global values +type GlobalINI struct { + // vCenter username. + User string `gcfg:"user"` + // vCenter password in clear text. + Password string `gcfg:"password"` + // Deprecated. Use VirtualCenter to specify multiple vCenter Servers. + // vCenter IP. + VCenterIP string `gcfg:"server"` + // vCenter port. + VCenterPort string `gcfg:"port"` + // True if vCenter uses self-signed cert. + InsecureFlag bool `gcfg:"insecure-flag"` + // Datacenter in which VMs are located. + Datacenters string `gcfg:"datacenters"` + // Soap round tripper count (retries = RoundTripper - 1) + RoundTripperCount uint `gcfg:"soap-roundtrip-count"` + // Specifies the path to a CA certificate in PEM format. Optional; if not + // configured, the system's CA certificates will be used. + CAFile string `gcfg:"ca-file"` + // Thumbprint of the VCenter's certificate thumbprint + Thumbprint string `gcfg:"thumbprint"` + // Name of the secret were vCenter credentials are present. + SecretName string `gcfg:"secret-name"` + // Secret Namespace where secret will be present that has vCenter credentials. + SecretNamespace string `gcfg:"secret-namespace"` + // Secret directory in the event that: + // 1) we don't want to use the k8s API to listen for changes to secrets + // 2) we are not in a k8s env, namely DC/OS, since CSI is CO agnostic + // Default: /etc/cloud/credentials + SecretsDirectory string `gcfg:"secrets-directory"` + // Disable the vSphere CCM API + // Default: true + APIDisable bool `gcfg:"api-disable"` + // Configurable vSphere CCM API port + // Default: 43001 + APIBinding string `gcfg:"api-binding"` + // IP Family enables the ability to support IPv4 or IPv6 + // Supported values are: + // ipv4 - IPv4 addresses only (Default) + // ipv6 - IPv6 addresses only + IPFamily string `gcfg:"ip-family"` } -// VirtualCenterConfig contains information used to access a remote vCenter +// VirtualCenterConfigINI contains information used to access a remote vCenter // endpoint. -type VirtualCenterConfig struct { +type VirtualCenterConfigINI struct { // vCenter username. User string `gcfg:"user"` // vCenter password in clear text. @@ -114,3 +108,21 @@ type VirtualCenterConfig struct { // IPFamilyPriority (intentionally not exposed via the config) the list/priority of IP versions IPFamilyPriority []string } + +// LabelsINI tags categories and tags which correspond to "built-in node labels: zones and region" +type LabelsINI struct { + Zone string `gcfg:"zone"` + Region string `gcfg:"region"` +} + +// CommonConfigINI is used to read and store information from the cloud configuration file +type CommonConfigINI struct { + // Global values... + Global GlobalINI + + // Virtual Center configurations + VirtualCenter map[string]*VirtualCenterConfigINI + + // Tag categories and tags which correspond to "built-in node labels: zones and region" + Labels LabelsINI +} diff --git a/pkg/common/config/types_yaml.go b/pkg/common/config/types_yaml.go new file mode 100644 index 000000000..472c279ec --- /dev/null +++ b/pkg/common/config/types_yaml.go @@ -0,0 +1,132 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +/* + TODO: + When the INI based cloud-config is deprecated, this file should be renamed + from types_yaml.go to types.go and the structs within this file should be named: + + GlobalYAML -> Global + VirtualCenterConfigYAML -> VirtualCenterConfig + LabelsYAML -> Labels + ConfigYAML -> Config +*/ + +// GlobalYAML are global values +type GlobalYAML struct { + // vCenter username. + User string `yaml:"user"` + // vCenter password in clear text. + Password string `yaml:"password"` + // Deprecated. Use VirtualCenter to specify multiple vCenter Servers. + // vCenter IP. + VCenterIP string `yaml:"server"` + // vCenter port. + VCenterPort uint `yaml:"port"` + // True if vCenter uses self-signed cert. + InsecureFlag bool `yaml:"insecureFlag"` + // Datacenter in which VMs are located. + Datacenters []string `yaml:"datacenters"` + // Soap round tripper count (retries = RoundTripper - 1) + RoundTripperCount uint `yaml:"soapRoundtripCount"` + // Specifies the path to a CA certificate in PEM format. Optional; if not + // configured, the system's CA certificates will be used. + CAFile string `yaml:"caFile"` + // Thumbprint of the VCenter's certificate thumbprint + Thumbprint string `yaml:"thumbprint"` + // Name of the secret were vCenter credentials are present. + SecretName string `yaml:"secretName"` + // Secret Namespace where secret will be present that has vCenter credentials. + SecretNamespace string `yaml:"secretNamespace"` + // Secret directory in the event that: + // 1) we don't want to use the k8s API to listen for changes to secrets + // 2) we are not in a k8s env, namely DC/OS, since CSI is CO agnostic + // Default: /etc/cloud/credentials + SecretsDirectory string `yaml:"secretsDirectory"` + // Disable the vSphere CCM API + // Default: true + APIDisable bool `yaml:"apiDisable"` + // Configurable vSphere CCM API port + // Default: 43001 + APIBinding string `yaml:"apiBinding"` + // IP Family enables the ability to support IPv4 or IPv6 + // Supported values are: + // ipv4 - IPv4 addresses only (Default) + // ipv6 - IPv6 addresses only + IPFamilyPriority []string `yaml:"ipFamily"` +} + +// VirtualCenterConfigYAML contains information used to access a remote vCenter +// endpoint. +type VirtualCenterConfigYAML struct { + // vCenter username. + User string `yaml:"user"` + // vCenter password in clear text. + Password string `yaml:"password"` + // TenantRef (intentionally not exposed via the config) is a unique tenant ref to + // be used in place of the vcServer as the primary connection key. If one label is set, + // all virtual center configs must have a unique label. + TenantRef string + // vCenterIP - If this field in the config is set, it is assumed then that value in [VirtualCenter ""] + // is now the TenantRef above and this field is the actual VCenterIP. Otherwise for backward + // compatibility, the value by default is the IP or FQDN of the vCenter Server. + VCenterIP string `yaml:"server"` + // vCenter port. + VCenterPort uint `yaml:"port"` + // True if vCenter uses self-signed cert. + InsecureFlag bool `yaml:"insecureFlag"` + // Datacenter in which VMs are located. + Datacenters []string `yaml:"datacenters"` + // Soap round tripper count (retries = RoundTripper - 1) + RoundTripperCount uint `yaml:"soapRoundtripCount"` + // Specifies the path to a CA certificate in PEM format. Optional; if not + // configured, the system's CA certificates will be used. + CAFile string `yaml:"caFile"` + // Thumbprint of the VCenter's certificate thumbprint + Thumbprint string `yaml:"thumbprint"` + // SecretRef (intentionally not exposed via the config) is a key to identify which + // InformerManager holds the secret + SecretRef string + // Name of the secret where vCenter credentials are present. + SecretName string `yaml:"secretName"` + // Namespace where the secret will be present containing vCenter credentials. + SecretNamespace string `yaml:"secretNamespace"` + // IP Family enables the ability to support IPv4 or IPv6 + // Supported values are: + // ipv4 - IPv4 addresses only (Default) + // ipv6 - IPv6 addresses only + IPFamilyPriority []string `yaml:"ipFamily"` +} + +// LabelsYAML tags categories and tags which correspond to "built-in node labels: zones and region" +type LabelsYAML struct { + Zone string `yaml:"zone"` + Region string `yaml:"region"` +} + +// CommonConfigYAML is used to read and store information from the cloud configuration file +type CommonConfigYAML struct { + // Global values... + Global GlobalYAML + + // Virtual Center configurations + Vcenter map[string]*VirtualCenterConfigYAML + + // Tag categories and tags which correspond to "built-in node labels: zones and region" + Labels LabelsYAML +} diff --git a/pkg/common/config/virtualcenterconfig.go b/pkg/common/config/virtualcenterconfig.go deleted file mode 100644 index a43afd484..000000000 --- a/pkg/common/config/virtualcenterconfig.go +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -// IsSecretInfoProvided returns true if the secret per VC has been configured -func (vcc *VirtualCenterConfig) IsSecretInfoProvided() bool { - return vcc.SecretName != "" && vcc.SecretNamespace != "" -} diff --git a/pkg/common/connectionmanager/list_test.go b/pkg/common/connectionmanager/list_test.go index 9950957a1..0ee6f561c 100644 --- a/pkg/common/connectionmanager/list_test.go +++ b/pkg/common/connectionmanager/list_test.go @@ -73,7 +73,6 @@ func configFromSimWithTLS(tlsConfig *tls.Config, insecureAllowed bool, multiDc b model.Service.RegisterSDK(lookup.New()) cfg.Global.InsecureFlag = insecureAllowed - cfg.Global.VCenterIP = s.URL.Hostname() cfg.Global.VCenterPort = s.URL.Port() cfg.Global.User = s.URL.User.Username() @@ -85,14 +84,15 @@ func configFromSimWithTLS(tlsConfig *tls.Config, insecureAllowed bool, multiDc b cfg.Global.Datacenters = vclib.TestDefaultDatacenter } cfg.VirtualCenter = make(map[string]*vcfg.VirtualCenterConfig) - cfg.VirtualCenter[s.URL.Hostname()] = &vcfg.VirtualCenterConfig{ - User: cfg.Global.User, - Password: cfg.Global.Password, - TenantRef: cfg.Global.VCenterIP, - VCenterIP: cfg.Global.VCenterIP, - VCenterPort: cfg.Global.VCenterPort, - InsecureFlag: cfg.Global.InsecureFlag, - Datacenters: cfg.Global.Datacenters, + cfg.VirtualCenter[cfg.Global.VCenterIP] = &vcfg.VirtualCenterConfig{ + User: cfg.Global.User, + Password: cfg.Global.Password, + TenantRef: cfg.Global.VCenterIP, + VCenterIP: cfg.Global.VCenterIP, + VCenterPort: cfg.Global.VCenterPort, + InsecureFlag: cfg.Global.InsecureFlag, + Datacenters: cfg.Global.Datacenters, + IPFamilyPriority: []string{vcfg.DefaultIPFamily}, } // Configure region and zone categories @@ -105,12 +105,13 @@ func configFromSimWithTLS(tlsConfig *tls.Config, insecureAllowed bool, multiDc b } } +// configFromEnvOrSim builds a config from configFromSim and overrides using configFromEnv func configFromEnvOrSim(multiDc bool) (*vcfg.Config, func()) { - cfg := &vcfg.Config{} + cfg, fin := configFromSim(multiDc) if err := cfg.FromEnv(); err != nil { - return configFromSim(multiDc) + return nil, nil } - return cfg, func() {} + return cfg, fin } func TestListAllVcPairs(t *testing.T) {