diff --git a/.gitignore b/.gitignore index b4f3e57c7d..5f9d754c64 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ verify-network *.iml .DS_Store portmap +tmp diff --git a/Makefile b/Makefile index 8dd9998fe8..b7ea24745a 100644 --- a/Makefile +++ b/Makefile @@ -85,3 +85,4 @@ clean: rm -f aws-k8s-agent rm -f aws-cni rm -f portmap + rm -rf tmp diff --git a/go.mod b/go.mod index cdc743cc79..a955f35c7d 100644 --- a/go.mod +++ b/go.mod @@ -3,58 +3,63 @@ module github.com/aws/amazon-vpc-cni-k8s go 1.12 require ( - github.com/Microsoft/go-winio v0.4.11 + github.com/Microsoft/go-winio v0.4.11 // indirect github.com/aws/aws-sdk-go v1.15.22 - github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 + github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect github.com/cihub/seelog v0.0.0-20151216151435-d2c6e5aa9fbf github.com/containernetworking/cni v0.5.2 github.com/coreos/go-iptables v0.4.0 - github.com/davecgh/go-spew v1.1.1 - github.com/docker/distribution v2.6.2+incompatible + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/deckarep/golang-set v1.7.1 + github.com/docker/distribution v2.6.2+incompatible // indirect github.com/docker/docker v1.13.1 - github.com/docker/go-connections v0.4.0 - github.com/docker/go-units v0.3.3 - github.com/ghodss/yaml v1.0.0 - github.com/go-ini/ini v1.38.2 - github.com/gogo/protobuf v1.1.1 - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.3.3 // indirect + github.com/ghodss/yaml v1.0.0 // indirect + github.com/go-ini/ini v1.38.2 // indirect + github.com/gogo/protobuf v1.1.1 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect github.com/golang/mock v1.1.1 github.com/golang/protobuf v1.2.0 - github.com/google/btree v1.0.0 - github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf - github.com/googleapis/gnostic v0.2.0 - github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc - github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 - github.com/imdario/mergo v0.3.6 - github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 - github.com/json-iterator/go v1.1.5 - github.com/matttproud/golang_protobuf_extensions v1.0.1 - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd - github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 + github.com/google/btree v1.0.0 // indirect + github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect + github.com/googleapis/gnostic v0.2.0 // indirect + github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect + github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/json-iterator/go v1.1.5 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect + github.com/onsi/ginkgo v1.8.0 // indirect + github.com/onsi/gomega v1.5.0 // indirect github.com/operator-framework/operator-sdk v0.0.7 - github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c - github.com/peterbourgon/diskv v2.0.1+incompatible + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.8.0 - github.com/pmezard/go-difflib v1.0.0 + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v0.8.0 - github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 - github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e - github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 - github.com/sirupsen/logrus v1.0.6 - github.com/spf13/pflag v1.0.2 + github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect + github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e // indirect + github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 // indirect + github.com/sirupsen/logrus v1.0.6 // indirect + github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect + github.com/spf13/pflag v1.0.2 // indirect + github.com/stevvooe/resumable v0.0.0-20180830230917-22b14a53ba50 // indirect github.com/stretchr/testify v1.2.2 github.com/vishvananda/netlink v1.0.0 - github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc - golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac - golang.org/x/net v0.0.0-20180826012351-8a410e7b638d - golang.org/x/sys v0.0.0-20180828065106-d99a578cf41b - golang.org/x/text v0.3.0 - golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 - google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 + github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect + golang.org/x/net v0.0.0-20190311183353-d8887717615a + golang.org/x/sync v0.0.0-20190412183630-56d357773e84 // indirect + golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a + golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect + google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 // indirect google.golang.org/grpc v1.14.0 - gopkg.in/inf.v0 v0.9.1 - gopkg.in/yaml.v2 v2.2.1 + gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect + gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.42.0 // indirect k8s.io/api v0.0.0-20180712090710-2d6f90ab1293 k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d k8s.io/client-go v0.0.0-20180806134042-1f13a808da65 + k8s.io/kube-openapi v0.0.0-20190418160015-6b3d3b2d5666 // indirect ) diff --git a/go.sum b/go.sum index b2bcb1d1fe..d790fb1988 100644 --- a/go.sum +++ b/go.sum @@ -1,56 +1,179 @@ +github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/aws/aws-sdk-go v1.15.22 h1:oBDjhvhppuHcEzchKrAB2tnt8nENQG47dGiC1865tqA= github.com/aws/aws-sdk-go v1.15.22/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/cihub/seelog v0.0.0-20151216151435-d2c6e5aa9fbf h1:XI2tOTCBqEnMyN2j1yPBI07yQHeywUSCEf8YWqf0oKw= github.com/cihub/seelog v0.0.0-20151216151435-d2c6e5aa9fbf/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= +github.com/containernetworking/cni v0.5.2 h1:/nFPNGJQu4yiNvXxH31qL05FIyGG5Y/p0YxuUutl+PE= github.com/containernetworking/cni v0.5.2/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/coreos/go-iptables v0.4.0 h1:wh4UbVs8DhLUbpyq97GLJDKrQMjEDD63T1xE4CrsKzQ= github.com/coreos/go-iptables v0.4.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ= +github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/docker/distribution v2.6.2+incompatible h1:4FI6af79dfCS/CYb+RRtkSHw3q1L/bnDjG1PcPZtQhM= github.com/docker/distribution v2.6.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.13.1 h1:IkZjBSIc8hBjLpqeAbeE5mca5mNgeatLHBy3GO78BWo= github.com/docker/docker v1.13.1/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-ini/ini v1.38.2 h1:6Hl/z3p3iFkA0dlDfzYxuFuUGD+kaweypF6btsR2/Q4= github.com/go-ini/ini v1.38.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 h1:UnszMmmmm5vLwWzDjTFVIkfhvWF1NdrmChl8L2NUDCw= github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/operator-framework/operator-sdk v0.0.7 h1:feujqHLhibLBbDVrSAFswpSzTVS5mEuarvywJ079mYE= github.com/operator-framework/operator-sdk v0.0.7/go.mod h1:iVyukRkam5JZa8AnjYf+/G3rk7JI1+M6GsU0sq0B9NA= -github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.8.0 h1:1921Yw9Gc3iSc4VQh3PIoOqgPCZS7G/4xQNVUp8Mda8= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e h1:n/3MEhJQjQxrOUCzh1Y3Re6aJUUWRp2M9+Oc3eVn/54= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 h1:agujYaXJSxSo18YNX3jzl+4G6Bstwt+kqv47GS12uL0= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/sirupsen/logrus v1.0.6 h1:hcP1GmhGigz/O7h1WVUM5KklBp1JoNS9FggWKdj/j3s= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stevvooe/resumable v0.0.0-20180830230917-22b14a53ba50 h1:4bT0pPowCpQImewr+BjzfUKcuFW+KVyB8d1OF3b6oTI= +github.com/stevvooe/resumable v0.0.0-20180830230917-22b14a53ba50/go.mod h1:1pdIZTAHUz+HDKDVZ++5xg/duPlhKAIzw9qy42CWYp4= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM= github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= -golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/sys v0.0.0-20180828065106-d99a578cf41b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84 h1:IqXQ59gzdXv58Jmm2xn0tSOR9i6HqroaOFRQ3wR/dJQ= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/grpc v1.14.0 h1:ArxJuB1NWfPY6r9Gp9gqwplT0Ge7nqv9msgu03lHLmo= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +k8s.io/api v0.0.0-20180712090710-2d6f90ab1293 h1:hROmpFC7JMobXFXMmD7ZKZLhDKvr1IKfFJoYS/45G/8= k8s.io/api v0.0.0-20180712090710-2d6f90ab1293/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d h1:MZjlsu9igBoVPZkXpIGoxI6EonqNsXXZU7hhvfQLkd4= k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/client-go v0.0.0-20180806134042-1f13a808da65 h1:kQX7jEIMYrWV9XqFN4usRaBLzCu7fd/qsCXxbgf3+9g= k8s.io/client-go v0.0.0-20180806134042-1f13a808da65/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/kube-openapi v0.0.0-20190418160015-6b3d3b2d5666 h1:hlzz2EvLPcefAcG/j0tOZpds4LWSElZzxpZuhxbblbc= +k8s.io/kube-openapi v0.0.0-20190418160015-6b3d3b2d5666/go.mod h1:jqYp7BKXW0Jl+F1dWXBieUmcHKMPpGHGWA0uqfpOZZ4= +sigs.k8s.io/structured-merge-diff v0.0.0-20181214233322-d43a45b8663b/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= diff --git a/ipamd/datastore/data_store.go b/ipamd/datastore/data_store.go index b32ebea1f2..01e3b22a5a 100644 --- a/ipamd/datastore/data_store.go +++ b/ipamd/datastore/data_store.go @@ -84,7 +84,7 @@ var ( // ENIIPPool contains ENI/IP Pool information. Exported fields will be marshaled for introspection. type ENIIPPool struct { createTime time.Time - lastUnAssignedTime time.Time + lastUnassignedTime time.Time // IsPrimary indicates whether ENI is a primary ENI IsPrimary bool ID string @@ -99,9 +99,9 @@ type ENIIPPool struct { // AddressInfo contains information about an IP, Exported fields will be marshaled for introspection. type AddressInfo struct { - address string + Address string Assigned bool // true if it is assigned to a pod - unAssignedTime time.Time + UnassignedTime time.Time } // PodKey is used to locate pod IP @@ -180,8 +180,8 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary bool) erro return nil } -// AddENIIPv4Address add an IP of an ENI to data store -func (ds *DataStore) AddENIIPv4Address(eniID string, ipv4 string) error { +// AddIPv4AddressToStore add an IP of an ENI to data store +func (ds *DataStore) AddIPv4AddressToStore(eniID string, ipv4 string) error { ds.lock.Lock() defer ds.lock.Unlock() @@ -202,13 +202,13 @@ func (ds *DataStore) AddENIIPv4Address(eniID string, ipv4 string) error { // Prometheus gauge totalIPs.Set(float64(ds.total)) - curENI.IPv4Addresses[ipv4] = &AddressInfo{address: ipv4, Assigned: false} + curENI.IPv4Addresses[ipv4] = &AddressInfo{Address: ipv4, Assigned: false} log.Infof("Added ENI(%s)'s IP %s to datastore", eniID, ipv4) return nil } -// DelENIIPv4Address delete an IP of ENI from datastore -func (ds *DataStore) DelENIIPv4Address(eniID string, ipv4 string) error { +// DelIPv4AddressFromStore delete an IP of ENI from datastore +func (ds *DataStore) DelIPv4AddressFromStore(eniID string, ipv4 string) error { ds.lock.Lock() defer ds.lock.Unlock() log.Debugf("Deleting ENI(%s)'s IPv4 address %s from datastore", eniID, ipv4) @@ -239,7 +239,7 @@ func (ds *DataStore) DelENIIPv4Address(eniID string, ipv4 string) error { } // AssignPodIPv4Address assigns an IPv4 address to pod -// It returns the assigned IPv4 address, device number, error +// It returns the assigned IPv4 Address, device number, error func (ds *DataStore) AssignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (string, int, error) { ds.lock.Lock() defer ds.lock.Unlock() @@ -259,7 +259,7 @@ func (ds *DataStore) AssignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (string, in return ipAddr.IP, ipAddr.DeviceNumber, nil } // TODO Handle this bug assert? May need to add a counter here, if counter is too high, need to mark node as unhealthy... - // This is a bug that the caller invokes multiple times to assign(PodName/NameSpace -> a different IP address). + // This is a bug that the caller invokes multiple times to assign(PodName/NameSpace -> a different IP Address). log.Errorf("AssignPodIPv4Address: current IP %s is changed to IP %s for pod(name %s, namespace %s, container %s)", ipAddr, k8sPod.IP, k8sPod.Name, k8sPod.Namespace, k8sPod.Container) return "", 0, errors.New("AssignPodIPv4Address: invalid pod with multiple IP addresses") @@ -282,23 +282,25 @@ func (ds *DataStore) assignPodIPv4AddressUnsafe(k8sPod *k8sapi.K8SPodInfo) (stri continue } for _, addr := range eni.IPv4Addresses { - if k8sPod.IP == addr.address { + if k8sPod.IP == addr.Address { // After L-IPAM restart and built IP warm-pool, it needs to take the existing running pod IP out of the pool. if !addr.Assigned { incrementAssignedCount(ds, eni, addr) } log.Infof("AssignPodIPv4Address: Reassign IP %v to pod (name %s, namespace %s)", - addr.address, k8sPod.Name, k8sPod.Namespace) - ds.podsIP[podKey] = PodIPInfo{IP: addr.address, DeviceNumber: eni.DeviceNumber} - return addr.address, eni.DeviceNumber, nil + addr.Address, k8sPod.Name, k8sPod.Namespace) + ds.podsIP[podKey] = PodIPInfo{IP: addr.Address, DeviceNumber: eni.DeviceNumber} + return addr.Address, eni.DeviceNumber, nil } - if !addr.Assigned && k8sPod.IP == "" && curTime.Sub(addr.unAssignedTime) > addressCoolingPeriod { + if !addr.Assigned && k8sPod.IP == "" && curTime.Sub(addr.UnassignedTime) > addressCoolingPeriod { // This is triggered by a pod's Add Network command from CNI plugin incrementAssignedCount(ds, eni, addr) log.Infof("AssignPodIPv4Address: Assign IP %v to pod (name %s, namespace %s container %s)", - addr.address, k8sPod.Name, k8sPod.Namespace, k8sPod.Container) - ds.podsIP[podKey] = PodIPInfo{IP: addr.address, DeviceNumber: eni.DeviceNumber} - return addr.address, eni.DeviceNumber, nil + addr.Address, k8sPod.Name, k8sPod.Namespace, k8sPod.Container) + ds.podsIP[podKey] = PodIPInfo{IP: addr.Address, DeviceNumber: eni.DeviceNumber} + return addr.Address, eni.DeviceNumber, nil + } else if curTime.Sub(addr.UnassignedTime) <= addressCoolingPeriod { + log.Debugf("AssignPodIPv4Address: IP %s is still in cooling period.", addr.Address) } } } @@ -329,7 +331,7 @@ func (ds *DataStore) getDeletableENI() *ENIIPPool { continue } - if time.Now().Sub(eni.lastUnAssignedTime) < addressENICoolingPeriod { + if time.Now().Sub(eni.lastUnassignedTime) < addressENICoolingPeriod { continue } @@ -385,8 +387,8 @@ func (ds *DataStore) RemoveUnusedENIFromDataStore() string { return removableENI } -// DeleteENI free a ENI. -func (ds *DataStore) DeleteENI(eni string) error { +// RemoveENIFromDataStore free a ENI. +func (ds *DataStore) RemoveENIFromDataStore(eni string) error { ds.lock.Lock() defer ds.lock.Unlock() @@ -401,7 +403,7 @@ func (ds *DataStore) DeleteENI(eni string) error { } ds.total -= len(eniIPPool.IPv4Addresses) - log.Infof("DeleteENI %s: IP address pool stats: free %d addresses, total: %d, assigned: %d", + log.Infof("RemoveENIFromDataStore %s: IP Address pool stats: free %d addresses, total: %d, assigned: %d", eni, len(eniIPPool.IPv4Addresses), ds.total, ds.assigned) delete(ds.eniIPPools, eni) @@ -415,7 +417,7 @@ func (ds *DataStore) DeleteENI(eni string) error { func (ds *DataStore) UnAssignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (string, int, error) { ds.lock.Lock() defer ds.lock.Unlock() - log.Debugf("UnAssignPodIPv4Address: IP address pool stats: total:%d, assigned %d, pod(Name: %s, Namespace: %s, Container %s)", + log.Debugf("UnAssignPodIPv4Address: IP Address pool stats: total:%d, assigned %d, pod(Name: %s, Namespace: %s, Container %s)", ds.total, ds.assigned, k8sPod.Name, k8sPod.Namespace, k8sPod.Container) podKey := PodKey{ @@ -438,12 +440,12 @@ func (ds *DataStore) UnAssignPodIPv4Address(k8sPod *k8sapi.K8SPodInfo) (string, assignedIPs.Set(float64(ds.assigned)) eni.AssignedIPv4Addresses-- curTime := time.Now() - ip.unAssignedTime = curTime - eni.lastUnAssignedTime = curTime + ip.UnassignedTime = curTime + eni.lastUnassignedTime = curTime log.Infof("UnAssignPodIPv4Address: pod (Name: %s, NameSpace %s Container %s)'s ipAddr %s, DeviceNumber%d", - k8sPod.Name, k8sPod.Namespace, k8sPod.Container, ip.address, eni.DeviceNumber) + k8sPod.Name, k8sPod.Namespace, k8sPod.Container, ip.Address, eni.DeviceNumber) delete(ds.podsIP, podKey) - return ip.address, eni.DeviceNumber, nil + return ip.Address, eni.DeviceNumber, nil } } @@ -486,8 +488,8 @@ func (ds *DataStore) GetENIInfos() *ENIInfos { return &eniInfos } -// GetENIs provides the number of ENI in the datastore -func (ds *DataStore) GetENIs() int { +// GetNumENIs provides the number of ENI in the datastore +func (ds *DataStore) GetNumENIs() int { ds.lock.Lock() defer ds.lock.Unlock() return len(ds.eniIPPools) diff --git a/ipamd/datastore/data_store_test.go b/ipamd/datastore/data_store_test.go index e062ed3a80..afe731a1a6 100644 --- a/ipamd/datastore/data_store_test.go +++ b/ipamd/datastore/data_store_test.go @@ -54,13 +54,13 @@ func TestDeleteENI(t *testing.T) { eniInfos := ds.GetENIInfos() assert.Equal(t, len(eniInfos.ENIIPPools), 3) - err = ds.DeleteENI("eni-2") + err = ds.RemoveENIFromDataStore("eni-2") assert.NoError(t, err) eniInfos = ds.GetENIInfos() assert.Equal(t, len(eniInfos.ENIIPPools), 2) - err = ds.DeleteENI("unknown-eni") + err = ds.RemoveENIFromDataStore("unknown-eni") assert.Error(t, err) eniInfos = ds.GetENIInfos() @@ -76,28 +76,28 @@ func TestAddENIIPv4Address(t *testing.T) { err = ds.AddENI("eni-2", 2, false) assert.NoError(t, err) - err = ds.AddENIIPv4Address("eni-1", "1.1.1.1") + err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") assert.NoError(t, err) assert.Equal(t, ds.total, 1) assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 1) - err = ds.AddENIIPv4Address("eni-1", "1.1.1.1") + err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") assert.Error(t, err) assert.Equal(t, ds.total, 1) assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 1) - err = ds.AddENIIPv4Address("eni-1", "1.1.1.2") + err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.2") assert.NoError(t, err) assert.Equal(t, ds.total, 2) assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) - err = ds.AddENIIPv4Address("eni-2", "1.1.2.2") + err = ds.AddIPv4AddressToStore("eni-2", "1.1.2.2") assert.NoError(t, err) assert.Equal(t, ds.total, 3) assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) assert.Equal(t, len(ds.eniIPPools["eni-2"].IPv4Addresses), 1) - err = ds.AddENIIPv4Address("dummy-eni", "1.1.2.2") + err = ds.AddIPv4AddressToStore("dummy-eni", "1.1.2.2") assert.Error(t, err) assert.Equal(t, ds.total, 3) assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) @@ -114,17 +114,17 @@ func TestGetENIIPPools(t *testing.T) { err = ds.AddENI("eni-2", 2, false) assert.NoError(t, err) - err = ds.AddENIIPv4Address("eni-1", "1.1.1.1") + err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") assert.NoError(t, err) assert.Equal(t, ds.total, 1) assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 1) - err = ds.AddENIIPv4Address("eni-1", "1.1.1.2") + err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.2") assert.NoError(t, err) assert.Equal(t, ds.total, 2) assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) - err = ds.AddENIIPv4Address("eni-2", "1.1.2.2") + err = ds.AddIPv4AddressToStore("eni-2", "1.1.2.2") assert.NoError(t, err) assert.Equal(t, ds.total, 3) assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) @@ -143,28 +143,28 @@ func TestDelENIIPv4Address(t *testing.T) { err := ds.AddENI("eni-1", 1, true) assert.NoError(t, err) - err = ds.AddENIIPv4Address("eni-1", "1.1.1.1") + err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") assert.NoError(t, err) assert.Equal(t, ds.total, 1) assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 1) - err = ds.AddENIIPv4Address("eni-1", "1.1.1.2") + err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.2") assert.NoError(t, err) assert.Equal(t, ds.total, 2) assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) - err = ds.AddENIIPv4Address("eni-1", "1.1.1.3") + err = ds.AddIPv4AddressToStore("eni-1", "1.1.1.3") assert.NoError(t, err) assert.Equal(t, ds.total, 3) assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 3) - err = ds.DelENIIPv4Address("eni-1", "1.1.1.2") + err = ds.DelIPv4AddressFromStore("eni-1", "1.1.1.2") assert.NoError(t, err) assert.Equal(t, ds.total, 2) assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) // delete a unknown IP - err = ds.DelENIIPv4Address("eni-1", "10.10.10.10") + err = ds.DelIPv4AddressFromStore("eni-1", "10.10.10.10") assert.Error(t, err) assert.Equal(t, ds.total, 2) assert.Equal(t, len(ds.eniIPPools["eni-1"].IPv4Addresses), 2) @@ -177,11 +177,11 @@ func TestPodIPv4Address(t *testing.T) { ds.AddENI("eni-2", 2, false) - ds.AddENIIPv4Address("eni-1", "1.1.1.1") + ds.AddIPv4AddressToStore("eni-1", "1.1.1.1") - ds.AddENIIPv4Address("eni-1", "1.1.1.2") + ds.AddIPv4AddressToStore("eni-1", "1.1.1.2") - ds.AddENIIPv4Address("eni-2", "1.1.2.2") + ds.AddIPv4AddressToStore("eni-2", "1.1.2.2") podInfo := k8sapi.K8SPodInfo{ Name: "pod-1", @@ -292,7 +292,7 @@ func TestPodIPv4Address(t *testing.T) { assert.True(t, eni == "") ds.eniIPPools["eni-2"].createTime = time.Time{} - ds.eniIPPools["eni-2"].lastUnAssignedTime = time.Time{} + ds.eniIPPools["eni-2"].lastUnassignedTime = time.Time{} eni = ds.RemoveUnusedENIFromDataStore() assert.Equal(t, eni, "eni-2") diff --git a/ipamd/introspect.go b/ipamd/introspect.go index ca7bc7e04a..ca8099f2a0 100644 --- a/ipamd/introspect.go +++ b/ipamd/introspect.go @@ -37,7 +37,7 @@ type rootResponse struct { } // LoggingHandler is a object for handling http request -type LoggingHandler struct{ +type LoggingHandler struct { h http.Handler } diff --git a/ipamd/ipamd.go b/ipamd/ipamd.go index 8e4d630637..a80b3818bf 100644 --- a/ipamd/ipamd.go +++ b/ipamd/ipamd.go @@ -14,6 +14,7 @@ package ipamd import ( + "fmt" "net" "os" "strconv" @@ -21,6 +22,7 @@ import ( "time" log "github.com/cihub/seelog" + set "github.com/deckarep/golang-set" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -45,6 +47,7 @@ const ( maxRetryCheckENI = 5 eniAttachTime = 10 * time.Second nodeIPPoolReconcileInterval = 60 * time.Second + decreaseIPPoolInterval = 30 * time.Second maxK8SRetries = 12 retryK8SInterval = 5 * time.Second @@ -150,12 +153,13 @@ type IPAMContext struct { currentMaxAddrsPerENI int64 maxAddrsPerENI int64 - // maxENI indicate the maximum number of ENIs can be attached to the instance + // maxENIs indicate the maximum number of ENIs can be attached to the instance // It is initialized to 0 and it is set to current number of ENIs attached // when ipamd receives AttachmentLimitExceeded error - maxENI int + maxENIs int primaryIP map[string]string lastNodeIPPoolAction time.Time + lastDecreaseIPPool time.Time } func prometheusRegister() { @@ -201,6 +205,7 @@ func New(k8sapiClient k8sapi.K8SAPIs, eniConfig *eniconfig.ENIConfigController) func (c *IPAMContext) nodeInit() error { ipamdActionsInprogress.WithLabelValues("nodeInit").Add(float64(1)) defer ipamdActionsInprogress.WithLabelValues("nodeInit").Sub(float64(1)) + log.Debugf("Start node init") instanceMaxENIs, _ := c.awsClient.GetENILimit() maxENIs := getMaxENI(instanceMaxENIs) @@ -324,7 +329,6 @@ func (c *IPAMContext) getLocalPodsWithRetry() ([]*k8sapi.K8SPodInfo, error) { time.Sleep(retryK8SInterval) } - // TODO consider using map for _, pod := range pods { // needs to find the container ID for _, container := range containers { @@ -348,177 +352,254 @@ func (c *IPAMContext) StartNodeIPPoolManager() { } func (c *IPAMContext) updateIPPoolIfRequired() { - c.retryAllocENIIP() if c.nodeIPPoolTooLow() { c.increaseIPPool() } else if c.nodeIPPoolTooHigh() { - c.decreaseIPPool() + c.decreaseIPPool(decreaseIPPoolInterval) } } -// TODO: Does not retry! -func (c *IPAMContext) retryAllocENIIP() { - ipamdActionsInprogress.WithLabelValues("retryAllocENIIP").Add(float64(1)) - defer ipamdActionsInprogress.WithLabelValues("retryAllocENIIP").Sub(float64(1)) +func (c *IPAMContext) decreaseIPPool(interval time.Duration) { + ipamdActionsInprogress.WithLabelValues("decreaseIPPool").Add(float64(1)) + defer ipamdActionsInprogress.WithLabelValues("decreaseIPPool").Sub(float64(1)) - curIPTarget, warmIPTargetDefined := c.getCurWarmIPTarget() - if warmIPTargetDefined && curIPTarget <= 0 { - log.Debugf("Skipping retry allocating ENI IP, warm IP target reached") + now := time.Now() + timeSinceLast := now.Sub(c.lastDecreaseIPPool) + if timeSinceLast <= interval { + log.Debugf("decreaseIPPool: skipping because time since last %v <= %v", timeSinceLast, interval) return } - maxIPLimit, err := c.awsClient.GetENIipLimit() - if err != nil { - log.Infof("Failed to retrieve ENI IP limit: %v", err) - return - } - eni := c.dataStore.GetENINeedsIP(maxIPLimit, UseCustomNetworkCfg()) - if eni != nil { - log.Debugf("Attempt again to allocate IP address for ENI :%s", eni.ID) - var err error - if warmIPTargetDefined { - err = c.awsClient.AllocIPAddresses(eni.ID, int64(curIPTarget)) - } else { - err = c.awsClient.AllocIPAddresses(eni.ID, maxIPLimit) - } - if err != nil { - ipamdErrInc("retryAllocENIIPAllocAllIPAddressFailed", err) - log.Warn("During eni repair: error encountered on allocate IP address", err) - return - } - ec2Addrs, _, err := c.getENIaddresses(eni.ID) + + log.Debugf("Start decreasing IP pool") + + // TODO(nckturner) shouldn't we attempt to remove more than 1 ENI at a time here? + eni := c.dataStore.RemoveUnusedENIFromDataStore() + if eni != "" { + log.Debugf("Start freeing ENI %s", eni) + err := c.awsClient.FreeENI(eni) if err != nil { - ipamdErrInc("retryAllocENIIPgetENIaddressesFailed", err) - log.Warn("During eni repair: failed to get ENI ip addresses", err) + ipamdErrInc("decreaseIPPoolFreeENIFailed", err) + log.Errorf("Failed to free ENI %s, err: %v", eni, err) return } - c.lastNodeIPPoolAction = time.Now() - c.addENIaddressesToDataStore(ec2Addrs, eni.ID) + } + log.Info("No ENI to remove, all ENIs have IPs in use") + + // Determine if there are IPs to free + // When we have extra IPs beyond the target and warmIPTargetDefined is enabled, deallocate extra IP addresses + if _, over, warmIPTargetDefined := c.ipTargetState(); warmIPTargetDefined && over > 0 { + eniInfos := c.dataStore.GetENIInfos() + for eniID := range eniInfos.ENIIPPools { + ips, err := c.findFreeableIPs(eniID) + if err != nil { + log.Errorf("Error finding unassigned IPs: %s", err) + return + } + + if len(ips) == 0 { + continue + } + + // Deallocate IPs from the instance if they aren't used by pods. + if err := c.awsClient.DeallocIPAddresses(eniID, ips); err != nil { + log.Debugf(fmt.Sprintf("Failed to decrease IP pool by removing IPs %v from ENI %s: %s", ips, eniID, err)) + } else { + for _, unassignedIP := range ips { + err := c.dataStore.DelIPv4AddressFromStore(eniID, unassignedIP) + if err != nil { + log.Errorf("Failed to delete IP %s on ENI %s from datastore: %s", unassignedIP, eni, err) + ipamdErrInc("decreaseIPPool", err) + continue + } + } + } - curIPTarget, warmIPTargetDefined := c.getCurWarmIPTarget() - if warmIPTargetDefined && curIPTarget <= 0 { - log.Debugf("Finish retry allocating ENI IP, warm IP target reached") - return } } -} -func (c *IPAMContext) decreaseIPPool() { - ipamdActionsInprogress.WithLabelValues("decreaseIPPool").Add(float64(1)) - defer ipamdActionsInprogress.WithLabelValues("decreaseIPPool").Sub(float64(1)) - eni := c.dataStore.RemoveUnusedENIFromDataStore() - if eni == "" { - log.Info("No ENI to remove, all ENIs have IPs in use") - return - } - log.Debugf("Start freeing ENI %s", eni) - err := c.awsClient.FreeENI(eni) - if err != nil { - ipamdErrInc("decreaseIPPoolFreeENIFailed", err) - log.Errorf("Failed to free ENI %s, err: %v", eni, err) - return - } - c.lastNodeIPPoolAction = time.Now() + c.lastDecreaseIPPool = now + c.lastNodeIPPoolAction = now total, used := c.dataStore.GetStats() log.Debugf("Successfully decreased IP pool") logPoolStats(int64(total), int64(used), c.currentMaxAddrsPerENI, c.maxAddrsPerENI) } +// findFreeableIPs finds and returns IPs that are not assigned to Pods but are attached +// to ENIs on the node. +func (c *IPAMContext) findFreeableIPs(eni string) ([]string, error) { + podIPInfos := c.dataStore.GetPodInfos() + usedIPs := set.NewSet() + allocatedIPs := set.NewSet() + + // Get IPs that are currently in use by pods + for _, pod := range *podIPInfos { + usedIPs.Add(pod.IP) + } + + // Get IPs that are currently attached to the instance + eniInfos := c.dataStore.GetENIInfos() + eniIPPools := eniInfos.ENIIPPools + + pool, ok := eniIPPools[eni] + if !ok { + return nil, fmt.Errorf("error finding available IPs: eni %s does not exist", eni) + } + + for _, ip := range pool.IPv4Addresses { + allocatedIPs.Add(ip.Address) + } + + availableIPs := allocatedIPs.Difference(usedIPs).ToSlice() + freeableIPs := []string{} + + // Free the number of ips `over` the warm IP target, unless `over` is greater than the number of available IPs on + // this ENI. In that case we should only free the number of available IPs. + _, over, _ := c.ipTargetState() + numFreeable := min(over, len(availableIPs)) + + for _, ip := range availableIPs[:numFreeable] { + freeableIPs = append(freeableIPs, ip.(string)) + } + return freeableIPs, nil +} + func isAttachmentLimitExceededError(err error) bool { return strings.Contains(err.Error(), "AttachmentLimitExceeded") } func (c *IPAMContext) increaseIPPool() { - log.Debug("Start increasing IP pool size") + log.Debug("Start increasing IP pool") ipamdActionsInprogress.WithLabelValues("increaseIPPool").Add(float64(1)) defer ipamdActionsInprogress.WithLabelValues("increaseIPPool").Sub(float64(1)) - curIPTarget, warmIPTargetDefined := c.getCurWarmIPTarget() - if warmIPTargetDefined && curIPTarget <= 0 { + short, _, warmIPTargetDefined := c.ipTargetState() + if warmIPTargetDefined && short == 0 { log.Debugf("Skipping increase IP pool, warm IP target reached") return } instanceMaxENIs, err := c.awsClient.GetENILimit() + if err != nil { + log.Errorf("Failed to get ENI limit due to unknown instance type %s", err) + } maxENIs := getMaxENI(instanceMaxENIs) if maxENIs >= 1 { enisMax.Set(float64(maxENIs)) } - if err == nil && maxENIs == c.dataStore.GetENIs() { - log.Debugf("Skipping increase IP pool due to max ENI already attached to the instance : %d", maxENIs) - return - } - if (c.maxENI > 0) && (c.maxENI == c.dataStore.GetENIs()) { - if c.maxENI < maxENIs { - errString := "desired: " + strconv.FormatInt(int64(maxENIs), 10) + "current: " + strconv.FormatInt(int64(c.maxENI), 10) + // If discovered maxENIs is set, and it is equal to our current number of ENIs, then our pool is at maximum capacity + if (c.maxENIs > 0) && (c.maxENIs == c.dataStore.GetNumENIs()) { + // If discovered max is less than maxENIs + if c.maxENIs < maxENIs { + errString := fmt.Sprintf("desired: %d, current: %d", maxENIs, c.maxENIs) ipamdErrInc("unExpectedMaxENIAttached", errors.New(errString)) } - log.Debugf("Skipping increase IP pool due to max ENI already attached to the instance : %d", c.maxENI) + log.Debugf("Skipping increase IP pool due to max ENI already attached to the instance : %d", c.maxENIs) return } - var securityGroups []*string - var subnet string - customNetworkCfg := UseCustomNetworkCfg() - if customNetworkCfg { - eniCfg, err := c.eniConfig.MyENIConfig() + // Attempt to increase IP pool by allocating one ENI + if c.dataStore.GetNumENIs() < maxENIs { + var securityGroups []*string + var subnet string + customNetworkCfg := UseCustomNetworkCfg() + if customNetworkCfg { + eniCfg, err := c.eniConfig.MyENIConfig() + if err != nil { + log.Errorf("Failed to get pod ENI config") + return + } + + log.Infof("ipamd: using custom network config: %v, %s", eniCfg.SecurityGroups, eniCfg.Subnet) + for _, sgID := range eniCfg.SecurityGroups { + log.Debugf("Found security-group id: %s", sgID) + securityGroups = append(securityGroups, aws.String(sgID)) + } + subnet = eniCfg.Subnet + } + + eniID, err := c.awsClient.AllocENI(customNetworkCfg, securityGroups, subnet) if err != nil { - log.Errorf("Failed to get pod ENI config") + log.Errorf("Failed to increase pool size due to not able to allocate ENI %v", err) + + if isAttachmentLimitExceededError(err) { + c.maxENIs = c.dataStore.GetNumENIs() + log.Infof("Discovered the instance max ENI allowed is: %d", c.maxENIs) + } + // TODO need to add health stats + ipamdErrInc("increaseIPPoolAllocENI", err) return } + log.Infof("Allocated eniID: %s", eniID) - log.Infof("ipamd: using custom network config: %v, %s", eniCfg.SecurityGroups, eniCfg.Subnet) - for _, sgID := range eniCfg.SecurityGroups { - log.Debugf("Found security-group id: %s", sgID) - securityGroups = append(securityGroups, aws.String(sgID)) + maxIPPerENI, err := c.awsClient.GetENIipLimit() + if err != nil { + log.Infof("Failed to retrieve ENI IP limit: %v", err) + return } - subnet = eniCfg.Subnet - } - eni, err := c.awsClient.AllocENI(customNetworkCfg, securityGroups, subnet) - if err != nil { - log.Errorf("Failed to increase pool size due to not able to allocate ENI %v", err) - - if isAttachmentLimitExceededError(err) { - c.maxENI = c.dataStore.GetENIs() - log.Infof("Discovered the instance max ENI allowed is: %d", c.maxENI) + eniInfos := c.dataStore.GetENIInfos() + eni := eniInfos.ENIIPPools[eniID] + err = c.awsClient.AllocIPAddresses(eniID, maxIPPerENI-int64(len(eni.IPv4Addresses))) + if err != nil { + log.Warnf("Failed to allocate all available ip addresses on an eni %s: %s", eni.ID, err) + // Continue to process the allocated IP addresses + ipamdErrInc("increaseIPPoolAllocAllIPAddressFailed", err) } - // TODO need to add health stats - ipamdErrInc("increaseIPPoolAllocENI", err) - return - } - maxIPLimit, err := c.awsClient.GetENIipLimit() - if err != nil { - log.Infof("Failed to retrieve ENI IP limit: %v", err) - return - } + eniMetadata, err := c.waitENIAttached(eniID) + if err != nil { + ipamdErrInc("increaseIPPoolWaitENIAttachedFailed", err) + log.Errorf("Failed to increase pool size: not able to discover attached ENI from metadata service %v", err) + return + } - if warmIPTargetDefined { - err = c.awsClient.AllocIPAddresses(eni, int64(curIPTarget)) + err = c.setupENI(eniID, eniMetadata) + if err != nil { + ipamdErrInc("increaseIPPoolSetupENIFailed", err) + log.Errorf("Failed to increase pool size: %v", err) + return + } } else { - err = c.awsClient.AllocIPAddresses(eni, maxIPLimit) - } - if err != nil { - log.Warnf("Failed to allocate all available ip addresses on an ENI %v", err) - // Continue to process the allocated IP addresses - ipamdErrInc("increaseIPPoolAllocAllIPAddressFailed", err) + log.Debugf("Skipping ENI allocation due to max ENI already attached to the instance : %d", maxENIs) } - eniMetadata, err := c.waitENIAttached(eni) - if err != nil { - ipamdErrInc("increaseIPPoolwaitENIAttachedFailed", err) - log.Errorf("Failed to increase pool size: not able to discover attached ENI from metadata service %v", err) - return - } + // For each ENI, try to fill in missing IPs + eniInfos := c.dataStore.GetENIInfos() + for _, eni := range eniInfos.ENIIPPools { - err = c.setupENI(eni, eniMetadata) - if err != nil { - ipamdErrInc("increaseIPPoolsetupENIFailed", err) - log.Errorf("Failed to increase pool size: %v", err) - return + maxIPPerENI, err := c.awsClient.GetENIipLimit() + if err != nil { + log.Infof("Failed to retrieve ENI IP limit: %v", err) + return + } + + skipPrimary := UseCustomNetworkCfg() + if skipPrimary && eni.IsPrimary { + log.Debugf("Skip the primary ENI for need IP check") + continue + } + if int64(len(eni.IPv4Addresses)) < maxIPPerENI { + log.Debugf("Found ENI %s that has less than the maximum number of IP addresses allocated: cur=%d, max=%d", + eni.ID, len(eni.IPv4Addresses), maxIPPerENI) + + err = c.awsClient.AllocIPAddresses(eni.ID, maxIPPerENI-int64(len(eni.IPv4Addresses))) + if err != nil { + log.Warnf("Failed to allocate all available ip addresses on an eni %s: %s", eni.ID, err) + } + + ec2Addrs, _, err := c.getENIaddresses(eni.ID) + if err != nil { + ipamdErrInc("increaseIPPoolGetENIaddressesFailed", err) + log.Warn("During eni repair: failed to get ENI ip addresses", err) + return + } + + c.addENIaddressesToDataStore(ec2Addrs, eni.ID) + } } + c.lastNodeIPPoolAction = time.Now() total, used := c.dataStore.GetStats() log.Debugf("Successfully increased IP pool") @@ -574,7 +655,7 @@ func (c *IPAMContext) addENIaddressesToDataStore(ec2Addrs []*ec2.NetworkInterfac primaryIP = aws.StringValue(ec2Addr.PrivateIpAddress) continue } - err := c.dataStore.AddENIIPv4Address(eni, aws.StringValue(ec2Addr.PrivateIpAddress)) + err := c.dataStore.AddIPv4AddressToStore(eni, aws.StringValue(ec2Addr.PrivateIpAddress)) if err != nil && err.Error() != datastore.DuplicateIPError { log.Warnf("Failed to increase IP pool, failed to add IP %s to data store", ec2Addr.PrivateIpAddress) // continue to add next address @@ -635,12 +716,12 @@ func (c *IPAMContext) waitENIAttached(eni string) (awsutils.ENIMetadata, error) } // getMaxENI returns the maximum number of ENIs for this instance, which is -// the lesser of the given lower bound (for example, the limit for the instance +// the lesser of the given upper bound (for example, the limit for the instance // type) and a value configured via the MAX_ENI environment variable. // // If the value configured via environment variable is 0 or less, it is -// ignored, and the lowerBound is returned. -func getMaxENI(lowerBound int) int { +// ignored, and the upperBound is returned. +func getMaxENI(upperBound int) int { inputStr, found := os.LookupEnv(envMaxENI) envMax := defaultMaxENI @@ -651,12 +732,12 @@ func getMaxENI(lowerBound int) int { } } - // If envMax is defined (>=1) and is less than the input lower bound, return + // If envMax is defined (>=1) and is less than the input upper bound, return // envMax. - if envMax >= 1 && envMax < lowerBound { + if envMax >= 1 && envMax < upperBound { return envMax } - return lowerBound + return upperBound } func getWarmENITarget() int { @@ -683,13 +764,9 @@ func logPoolStats(total, used, currentMaxAddrsPerENI, maxAddrsPerENI int64) { // nodeIPPoolTooLow returns true if IP pool is below low threshold func (c *IPAMContext) nodeIPPoolTooLow() bool { - curIPTarget, warmIPTargetDefined := c.getCurWarmIPTarget() - if warmIPTargetDefined && curIPTarget <= 0 { - return false - } - - if warmIPTargetDefined && curIPTarget > 0 { - return true + short, _, warmIPTargetDefined := c.ipTargetState() + if warmIPTargetDefined { + return short > 0 } // If WARM-IP-TARGET not defined fallback using number of ENIs @@ -710,8 +787,9 @@ func (c *IPAMContext) nodeIPPoolTooHigh() bool { available := total - used target := getWarmIPTarget() - if int64(target) != noWarmIPTarget && int64(target) >= int64(available) { - return false + if int64(target) != noWarmIPTarget { + // When our available IPs are greater than our target, our pool is too high + return int64(available) > int64(target) } return int64(available) >= (int64(warmENITarget)+1)*c.maxAddrsPerENI @@ -721,15 +799,16 @@ func ipamdErrInc(fn string, err error) { ipamdErr.With(prometheus.Labels{"fn": fn, "error": err.Error()}).Inc() } -// nodeIPPoolReconcile reconcile ENI and IP info from metadata service and IP addresses in datastore +// nodeIPPoolReconcile Adds ENIs to the store that are present on the host, and removes ENIs from the store that are not +// present on the host. func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { ipamdActionsInprogress.WithLabelValues("nodeIPPoolReconcile").Add(float64(1)) defer ipamdActionsInprogress.WithLabelValues("nodeIPPoolReconcile").Sub(float64(1)) curTime := time.Now() - last := c.lastNodeIPPoolAction - - if curTime.Sub(last) <= interval { + timeSinceLast := curTime.Sub(c.lastNodeIPPoolAction) + if time.Duration(timeSinceLast) <= interval { + log.Debugf("nodeIPPoolReconcile: skipping because time since last %v <= %v", timeSinceLast, interval) return } @@ -737,7 +816,7 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { attachedENIs, err := c.awsClient.GetAttachedENIs() if err != nil { - log.Error("IP pool reconcile: Failed to get attached ENI info", err.Error()) + log.Errorf("IP pool reconcile: Failed to get attached ENI info: %s", err.Error()) ipamdErrInc("reconcileFailedGetENIs", err) return } @@ -748,9 +827,8 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { for _, attachedENI := range attachedENIs { eniIPPool, err := c.dataStore.GetENIIPPools(attachedENI.ENIID) if err == nil { - // If the attached ENI is in the data store - log.Debugf("Reconcile existing ENI %s IP pool", attachedENI.ENIID) - // Reconcile IP pool + // Attached ENI is in the data store, reconcile its IPs + log.Debugf("IP pool reconcile: reconcile existing ENI %s IP pool", attachedENI.ENIID) c.eniIPPoolReconcile(eniIPPool, attachedENI, attachedENI.ENIID) // Mark action, remove this ENI from curENIs list delete(curENIs.ENIIPPools, attachedENI.ENIID) @@ -758,12 +836,11 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { } // Add new ENI - log.Debugf("Reconcile and add a new ENI %s", attachedENI) + log.Debugf("IP pool reconcile: add a new ENI %s to the data store", attachedENI) err = c.setupENI(attachedENI.ENIID, attachedENI) if err != nil { log.Errorf("IP pool reconcile: Failed to setup ENI %s network: %v", attachedENI.ENIID, err) ipamdErrInc("eniReconcileAdd", err) - // Continue if having trouble with ONLY 1 ENI, instead of bailout here? continue } reconcileCnt.With(prometheus.Labels{"fn": "eniReconcileAdd"}).Inc() @@ -771,8 +848,8 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { // Sweep phase: since the marked ENI have been removed, the remaining ones needs to be sweeped for eni := range curENIs.ENIIPPools { - log.Infof("Reconcile and delete detached ENI %s", eni) - err = c.dataStore.DeleteENI(eni) + log.Infof("IP pool reconcile: delete detached ENI %s from data store", eni) + err = c.dataStore.RemoveENIFromDataStore(eni) if err != nil { log.Errorf("IP pool reconcile: Failed to delete ENI during reconcile: %v", err) ipamdErrInc("eniReconcileDel", err) @@ -780,27 +857,29 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { } reconcileCnt.With(prometheus.Labels{"fn": "eniReconcileDel"}).Inc() } - log.Debug("Successfully Reconciled ENI/IP pool") + log.Debug("IP pool reconcile: successfully Reconciled ENI/IP pool") c.lastNodeIPPoolAction = curTime } +// eniIPPoolReconcile Adds IPs to the store that are present on the ENI, and removes IPs from the store that are not +// present on the ENI. func (c *IPAMContext) eniIPPoolReconcile(ipPool map[string]*datastore.AddressInfo, attachedENI awsutils.ENIMetadata, eni string) { for _, localIP := range attachedENI.LocalIPv4s { if localIP == c.primaryIP[eni] { - log.Debugf("Reconcile and skip primary IP %s on ENI %s", localIP, eni) + log.Debugf("IP pool reconcile: reconcile and skip primary IP %s on ENI %s", localIP, eni) continue } - err := c.dataStore.AddENIIPv4Address(eni, localIP) + err := c.dataStore.AddIPv4AddressToStore(eni, localIP) if err != nil && err.Error() == datastore.DuplicateIPError { - log.Debugf("Reconciled IP %s on ENI %s", localIP, eni) + log.Debugf("IP pool reconcile: IP %s on ENI %s already present in store", localIP, eni) // mark action = remove it from eniPool delete(ipPool, localIP) continue } if err != nil { - log.Errorf("Failed to reconcile IP %s on ENI %s", localIP, eni) + log.Errorf("IP pool reconcile: failed to reconcile IP %s on ENI %s", localIP, eni) ipamdErrInc("ipReconcileAdd", err) // continue instead of bailout due to one ip continue @@ -810,10 +889,10 @@ func (c *IPAMContext) eniIPPoolReconcile(ipPool map[string]*datastore.AddressInf // Sweep phase, delete remaining IPs for existingIP := range ipPool { - log.Debugf("Reconcile and delete IP %s on ENI %s", existingIP, eni) - err := c.dataStore.DelENIIPv4Address(eni, existingIP) + log.Debugf("IP pool reconcile: reconcile and delete IP %s on ENI %s", existingIP, eni) + err := c.dataStore.DelIPv4AddressFromStore(eni, existingIP) if err != nil { - log.Errorf("Failed to reconcile and delete IP %s on ENI %s, %v", existingIP, eni, err) + log.Errorf("IP pool reconcile: failed to reconcile and delete IP %s on ENI %s, %v", existingIP, eni, err) ipamdErrInc("ipReconcileDel", err) // continue instead of bailout due to one ip continue @@ -850,17 +929,24 @@ func getWarmIPTarget() int { return noWarmIPTarget } -func (c *IPAMContext) getCurWarmIPTarget() (int64, bool) { +func (c *IPAMContext) ipTargetState() (short int, over int, enabled bool) { target := getWarmIPTarget() if target == noWarmIPTarget { // there is no WARM_IP_TARGET defined, fallback to use all IP addresses on ENI - return int64(target), false + return 0, 0, false } - total, used := c.dataStore.GetStats() - curTarget := int64(target) - int64(total-used) - log.Debugf("Current warm IP stats: target: %d, total: %d, used: %d, curTarget: %d", target, total, used, curTarget) - return curTarget, true + total, assigned := c.dataStore.GetStats() + available := total - assigned + + // short is greater than 0 when we have fewer available IPs than the warm IP target + short = max(target-available, 0) + + // over is the number of available IPs we have beyond the warm IP target + over = max(available-target, 0) + + log.Debugf("Current warm IP stats: target: %d, total: %d, assigned: %d, available: %d, short: %d, over %d", target, total, assigned, available, short, over) + return short, over, true } // GetConfigForDebug returns the active values of the configuration env vars (for debugging purposes). @@ -871,3 +957,19 @@ func GetConfigForDebug() map[string]interface{} { envCustomNetworkCfg: UseCustomNetworkCfg(), } } + +// max returns the larger of x or y. +func max(x, y int) int { + if x < y { + return y + } + return x +} + +// Max returns the larger of x or y. +func min(x, y int) int { + if y < x { + return y + } + return x +} diff --git a/ipamd/ipamd_test.go b/ipamd/ipamd_test.go index 01738b0644..494b938fe5 100644 --- a/ipamd/ipamd_test.go +++ b/ipamd/ipamd_test.go @@ -410,29 +410,29 @@ func TestGetCurWarmIPTarget(t *testing.T) { mockContext.dataStore = datastore.NewDataStore() os.Unsetenv("WARM_IP_TARGET") - _, warmIPTargetDefined := mockContext.getCurWarmIPTarget() + _, warmIPTargetDefined := mockContext.ipTargetState() assert.False(t, warmIPTargetDefined) os.Setenv("WARM_IP_TARGET", "5") - curWarmIPTarget, warmIPTargetDefined := mockContext.getCurWarmIPTarget() + curWarmIPTarget, warmIPTargetDefined := mockContext.ipTargetState() assert.True(t, warmIPTargetDefined) assert.Equal(t, curWarmIPTarget, int64(5)) // add 2 addresses to datastore mockContext.dataStore.AddENI("eni-1", 1, true) - mockContext.dataStore.AddENIIPv4Address("eni-1", "1.1.1.1") - mockContext.dataStore.AddENIIPv4Address("eni-1", "1.1.1.2") + mockContext.dataStore.AddIPv4AddressToStore("eni-1", "1.1.1.1") + mockContext.dataStore.AddIPv4AddressToStore("eni-1", "1.1.1.2") - curWarmIPTarget, warmIPTargetDefined = mockContext.getCurWarmIPTarget() + curWarmIPTarget, warmIPTargetDefined = mockContext.ipTargetState() assert.True(t, warmIPTargetDefined) assert.Equal(t, curWarmIPTarget, int64(3)) // add 3 more addresses to datastore - mockContext.dataStore.AddENIIPv4Address("eni-1", "1.1.1.3") - mockContext.dataStore.AddENIIPv4Address("eni-1", "1.1.1.4") - mockContext.dataStore.AddENIIPv4Address("eni-1", "1.1.1.5") + mockContext.dataStore.AddIPv4AddressToStore("eni-1", "1.1.1.3") + mockContext.dataStore.AddIPv4AddressToStore("eni-1", "1.1.1.4") + mockContext.dataStore.AddIPv4AddressToStore("eni-1", "1.1.1.5") - curWarmIPTarget, warmIPTargetDefined = mockContext.getCurWarmIPTarget() + curWarmIPTarget, warmIPTargetDefined = mockContext.ipTargetState() assert.True(t, warmIPTargetDefined) assert.Equal(t, curWarmIPTarget, int64(0)) } diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index 7e29f4faef..451f2a4d09 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -14,6 +14,7 @@ package awsutils import ( + "context" "fmt" "os" "strconv" @@ -114,6 +115,9 @@ type APIs interface { // AllocIPAddresses allocates numIPs IP addresses on a ENI AllocIPAddresses(eniID string, numIPs int64) error + // DeallocIPAddresses deallocates the list of IP addresses from a ENI + DeallocIPAddresses(eniID string, ips []string) error + // GetVPCIPv4CIDR returns VPC's 1st CIDR GetVPCIPv4CIDR() string @@ -131,7 +135,7 @@ type APIs interface { // GetENILimit returns the number of ENIs that can be attached to an instance GetENILimit() (int, error) - + // GetPrimaryENImac returns the mac address of the primary ENI GetPrimaryENImac() string } @@ -792,7 +796,7 @@ func (cache *EC2InstanceMetadataCache) deleteENI(eniName string) error { func (cache *EC2InstanceMetadataCache) DescribeENI(eniID string) ([]*ec2.NetworkInterfacePrivateIpAddress, *string, error) { eniIds := make([]*string, 0) eniIds = append(eniIds, aws.String(eniID)) - input := &ec2.DescribeNetworkInterfacesInput{ NetworkInterfaceIds: eniIds} + input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: eniIds} start := time.Now() result, err := cache.ec2SVC.DescribeNetworkInterfaces(input) @@ -847,8 +851,7 @@ func (cache *EC2InstanceMetadataCache) GetENILimit() (int, error) { eniLimit, ok := InstanceENIsAvailable[cache.instanceType] if !ok { - log.Errorf("Failed to get ENI limit due to unknown instance type %s", cache.instanceType) - return 0, errors.New(UnknownInstanceType) + return 0, errors.New(fmt.Sprintf("%s: %s", UnknownInstanceType, cache.instanceType)) } return eniLimit, nil } @@ -883,6 +886,34 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddresses(eniID string, numIPs int return nil } +// DeallocIPAddresses allocates numIPs of IP address on an ENI +func (cache *EC2InstanceMetadataCache) DeallocIPAddresses(eniID string, ips []string) error { + ctx := context.Background() + + log.Infof("Trying to unassign the following IPs %s from ENI %s", ips, eniID) + + ipsInput := []*string{} + for _, ip := range ips { + ipsInput = append(ipsInput, aws.String(ip)) + } + + input := &ec2.UnassignPrivateIpAddressesInput{ + NetworkInterfaceId: aws.String(eniID), + PrivateIpAddresses: ipsInput, + } + + start := time.Now() + _, err := cache.ec2SVC.UnassignPrivateIpAddressesWithContext(ctx, input) + awsAPILatency.WithLabelValues("UnassignPrivateIpAddressesWithContext", fmt.Sprint(err != nil)).Observe(msSince(start)) + if err != nil { + awsAPIErrInc("UnassignPrivateIpAddressesWithContext", err) + + log.Errorf("Failed to deallocate a private IP address %v", err) + return errors.Wrap(err, fmt.Sprintf("deallocate IP addresses: failed to deallocate private IP addresses: %s", ips)) + } + return nil +} + // AllocAllIPAddress allocates all IP addresses available on ENI (TODO: Cleanup) func (cache *EC2InstanceMetadataCache) AllocAllIPAddress(eniID string) error { log.Infof("Trying to allocate all available IP addresses on ENI: %s", eniID) diff --git a/pkg/ec2wrapper/client.go b/pkg/ec2wrapper/client.go index 080bfd32ae..5eef11c829 100644 --- a/pkg/ec2wrapper/client.go +++ b/pkg/ec2wrapper/client.go @@ -14,6 +14,8 @@ package ec2wrapper import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/aws/session" ec2svc "github.com/aws/aws-sdk-go/service/ec2" ) @@ -25,6 +27,7 @@ type EC2 interface { DeleteNetworkInterface(input *ec2svc.DeleteNetworkInterfaceInput) (*ec2svc.DeleteNetworkInterfaceOutput, error) DetachNetworkInterface(input *ec2svc.DetachNetworkInterfaceInput) (*ec2svc.DetachNetworkInterfaceOutput, error) AssignPrivateIpAddresses(input *ec2svc.AssignPrivateIpAddressesInput) (*ec2svc.AssignPrivateIpAddressesOutput, error) + UnassignPrivateIpAddressesWithContext(ctx aws.Context, input *ec2svc.UnassignPrivateIpAddressesInput, opts ...request.Option) (*ec2svc.UnassignPrivateIpAddressesOutput, error) DescribeNetworkInterfaces(input *ec2svc.DescribeNetworkInterfacesInput) (*ec2svc.DescribeNetworkInterfacesOutput, error) ModifyNetworkInterfaceAttribute(input *ec2svc.ModifyNetworkInterfaceAttributeInput) (*ec2svc.ModifyNetworkInterfaceAttributeOutput, error) CreateTags(input *ec2svc.CreateTagsInput) (*ec2svc.CreateTagsOutput, error) diff --git a/pkg/k8sapi/discovery.go b/pkg/k8sapi/discovery.go index cafbd459cc..5e2f741160 100644 --- a/pkg/k8sapi/discovery.go +++ b/pkg/k8sapi/discovery.go @@ -151,8 +151,7 @@ func (d *Controller) K8SGetLocalPodIPs() ([]*K8SPodInfo, error) { defer d.workerPodsLock.Unlock() for _, pod := range d.workerPods { - log.Infof("K8SGetLocalPodIPs discovered local Pods: %s %s %s %s", - pod.Name, pod.Namespace, pod.IP, pod.UID) + log.Infof("K8SGetLocalPodIPs discovered local Pods: %s %s %s %s", pod.Name, pod.Namespace, pod.IP, pod.UID) localPods = append(localPods, pod) } diff --git a/plugins/routed-eni/cni.go b/plugins/routed-eni/cni.go index 25a0d8139d..145f719f7a 100644 --- a/plugins/routed-eni/cni.go +++ b/plugins/routed-eni/cni.go @@ -142,7 +142,7 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap K8S_POD_NAME: string(k8sArgs.K8S_POD_NAME), K8S_POD_NAMESPACE: string(k8sArgs.K8S_POD_NAMESPACE), K8S_POD_INFRA_CONTAINER_ID: string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), - IfName: args.IfName}) + IfName: args.IfName}) if err != nil { log.Errorf("Error received from AddNetwork grpc call for pod %s namespace %s container %s: %v", diff --git a/plugins/routed-eni/cni_test.go b/plugins/routed-eni/cni_test.go index 10a4c24fdf..95c8a961da 100644 --- a/plugins/routed-eni/cni_test.go +++ b/plugins/routed-eni/cni_test.go @@ -33,14 +33,14 @@ import ( ) const ( - containerID = "test-container" - netNS = "/proc/ns/1234" - ifName = "eth0" - cniVersion = "1.0" - cniName = "aws-cni" - cniType = "aws-cni" - ipAddr = "10.0.1.15" - devNum = 4 + containerID = "test-container" + netNS = "/proc/ns/1234" + ifName = "eth0" + cniVersion = "1.0" + cniName = "aws-cni" + cniType = "aws-cni" + ipAddr = "10.0.1.15" + devNum = 4 ) func setup(t *testing.T) (*gomock.Controller,