From 456f58f98e4bfac305c7c84fcdbb0f56e0cdc54c Mon Sep 17 00:00:00 2001 From: "vito.he" Date: Wed, 30 Oct 2019 21:32:55 +0800 Subject: [PATCH 01/61] Release 1.2.0 --- CHANGE.md | 27 +++++++++++++++++++++++++++ go.sum | 4 ---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/CHANGE.md b/CHANGE.md index cdfca4fb6d..99b609e5b4 100644 --- a/CHANGE.md +++ b/CHANGE.md @@ -1,5 +1,32 @@ # Release Notes +## 1.2.0 + +### New Features + +- Move callService to invoker & support attachments +- Support dynamic config center which compatible with dubbo 2.6.x & 2.7.x and commit the zookeeper impl +- Add service token authorization support +- Add accessLogFilter support +- Delete example in dubbo-go project +- Add tps limit support +- Add execute limit support + +### Enhancement + +- Split gettyRPCClient.close and gettyRPCClientPool.remove in protocol/dubbo/pool.go +- Remove client from pool before closing it +- Enhance the logic for fetching the local address +- Add protocol_conf default values + +### Bugfixes + +- GettyRPCClientPool remove deadlock +- Fix failover cluster bug and url parameter retries change int to string type +- Fix url params unsafe map +- Read protocol config by map key in config yaml instead of protocol name +- Fix dubbo group issues #238/ + ## 1.1.0 ### New Features diff --git a/go.sum b/go.sum index bcde5b1f80..b730b78684 100644 --- a/go.sum +++ b/go.sum @@ -37,8 +37,6 @@ github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190802083043-4cd0c391755e/go.mod github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/apache/dubbo-go-hessian2 v1.2.5-0.20191029001541-894e45c9aaaa h1:11TO1wiM5bvGAVrmfN5atD8gZqUSPE1TBoIs8sI6Abk= github.com/apache/dubbo-go-hessian2 v1.2.5-0.20191029001541-894e45c9aaaa/go.mod h1:LWnndnrFXZmJLAzoyNAPNHSIJ1KOHVkTSsHgC3YYWlo= -github.com/apache/dubbo-go-hessian2 v1.3.0 h1:ZhQYDm8GHqIp6i53T4ZJHQBN11nAYAjxlwoVznfyvD8= -github.com/apache/dubbo-go-hessian2 v1.3.0/go.mod h1:LWnndnrFXZmJLAzoyNAPNHSIJ1KOHVkTSsHgC3YYWlo= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -104,8 +102,6 @@ github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dubbogo/getty v1.3.0 h1:GImOCANdts7dlRqi9GMVsZJnfst9EPyjTVTR1AesOD8= -github.com/dubbogo/getty v1.3.0/go.mod h1:K4b3MkGLf7T+lMgQNFgpg0dI1Wvv1PTisFs1Psf86kU= github.com/dubbogo/getty v1.3.1 h1:9fehwTo/D6+z6/+kADMbhbKeMkP80o/3g+XwV5lFLTY= github.com/dubbogo/getty v1.3.1/go.mod h1:dtLOEb1v6EMHsQNYRWEACiRLmTWB2kJGUAj1aXayPOg= github.com/dubbogo/gost v1.1.1 h1:JCM7vx5edPIjDA5ovJTuzEEXuw2t7xLyrlgi2mi5jHI= From daa1fc33d370caa3322eeae777381fbcb3e2649b Mon Sep 17 00:00:00 2001 From: "vito.he" Date: Wed, 30 Oct 2019 21:35:23 +0800 Subject: [PATCH 02/61] Release 1.2.0 --- CHANGE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGE.md b/CHANGE.md index 99b609e5b4..3558e34c63 100644 --- a/CHANGE.md +++ b/CHANGE.md @@ -8,7 +8,7 @@ - Support dynamic config center which compatible with dubbo 2.6.x & 2.7.x and commit the zookeeper impl - Add service token authorization support - Add accessLogFilter support -- Delete example in dubbo-go project +- Move example in dubbo-go project away - Add tps limit support - Add execute limit support From 11878c4d265a08013da96c5eb93aa60cdc29b0d1 Mon Sep 17 00:00:00 2001 From: "vito.he" Date: Wed, 30 Oct 2019 22:04:00 +0800 Subject: [PATCH 03/61] Release 1.2.0 --- CHANGE.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/CHANGE.md b/CHANGE.md index 3558e34c63..947a695ca8 100644 --- a/CHANGE.md +++ b/CHANGE.md @@ -4,13 +4,19 @@ ### New Features -- Move callService to invoker & support attachments -- Support dynamic config center which compatible with dubbo 2.6.x & 2.7.x and commit the zookeeper impl +- Add etcdv3 registry support +- Add nacos registry support +- Add fail fast cluster support +- Add available cluster support +- Add broadcast cluster support +- Add forking cluster support - Add service token authorization support -- Add accessLogFilter support -- Move example in dubbo-go project away +- Add accessLog filter support - Add tps limit support - Add execute limit support +- Move callService to invoker & support attachments +- Move example in dubbo-go project away +- Support dynamic config center which compatible with dubbo 2.6.x & 2.7.x and commit the zookeeper impl ### Enhancement @@ -18,6 +24,9 @@ - Remove client from pool before closing it - Enhance the logic for fetching the local address - Add protocol_conf default values +- Add task pool for getty +- Update getty: remove read queue +- Clean heartbeat from PendingResponse ### Bugfixes @@ -26,6 +35,8 @@ - Fix url params unsafe map - Read protocol config by map key in config yaml instead of protocol name - Fix dubbo group issues #238/ +- Fix bug in reference_config +- Fix high memory bug in zookeeper listener ## 1.1.0 From 55ab09ebf406ea8666b1c14b725708741f2be654 Mon Sep 17 00:00:00 2001 From: scott Date: Fri, 20 Dec 2019 17:22:00 +0800 Subject: [PATCH 04/61] Fix kubernetes import block and make map param --- go.mod | 6 + go.sum | 32 ++ registry/kubernetes/listener.go | 100 +++++ registry/kubernetes/listener_test.go | 1 + registry/kubernetes/registry.go | 351 +++++++++++++++ registry/kubernetes/registry_test.go | 1 + remoting/kubernetes/client.go | 638 +++++++++++++++++++++++++++ remoting/kubernetes/facade.go | 77 ++++ remoting/kubernetes/listener.go | 188 ++++++++ remoting/kubernetes/store.go | 305 +++++++++++++ remoting/kubernetes/store_test.go | 66 +++ 11 files changed, 1765 insertions(+) create mode 100644 registry/kubernetes/listener.go create mode 100644 registry/kubernetes/listener_test.go create mode 100644 registry/kubernetes/registry.go create mode 100644 registry/kubernetes/registry_test.go create mode 100644 remoting/kubernetes/client.go create mode 100644 remoting/kubernetes/facade.go create mode 100644 remoting/kubernetes/listener.go create mode 100644 remoting/kubernetes/store.go create mode 100644 remoting/kubernetes/store_test.go diff --git a/go.mod b/go.mod index 4d1f8acbba..fb972ebd03 100644 --- a/go.mod +++ b/go.mod @@ -50,4 +50,10 @@ require ( go.uber.org/zap v1.10.0 google.golang.org/grpc v1.22.1 gopkg.in/yaml.v2 v2.2.2 + k8s.io/api v0.0.0-20190325185214-7544f9db76f6 + k8s.io/apimachinery v0.0.0-20190223001710-c182ff3b9841 + k8s.io/client-go v8.0.0+incompatible + k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a // indirect ) + +go 1.13 diff --git a/go.sum b/go.sum index b730b78684..783d51232a 100644 --- a/go.sum +++ b/go.sum @@ -14,10 +14,13 @@ github.com/Jeffail/gabs v1.1.0 h1:kw5zCcl9tlJNHTDme7qbi21fDHZmXrnjMoXos3Jw/NI= github.com/Jeffail/gabs v1.1.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= github.com/Microsoft/go-winio v0.4.3 h1:M3NHMuPgMSUPdE5epwNUHlRPSVzHs8HpRTrVXhR0myo= github.com/Microsoft/go-winio v0.4.3/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.0.1 h1:iLrQrdwjDd52kHDA5op2UBJFjmOb9g+7scBan4RN8F0= github.com/NYTimes/gziphandler v1.0.1/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/SAP/go-hdb v0.12.0 h1:5hBQZ2jjyZ268qjDmoDZJuCyLzR6oRLI60eYzmTW9m4= github.com/SAP/go-hdb v0.12.0/go.mod h1:etBT+FAi1t5k3K3tf5vQTnosgYmhDkRi8jEnQqCnxF0= github.com/SermoDigital/jose v0.0.0-20180104203859-803625baeddc h1:LkkwnbY+S8WmwkWq1SVyRWMH9nYWO1P5XN3OD1tts/w= @@ -86,6 +89,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/creasty/defaults v1.3.0 h1:uG+RAxYbJgOPCOdKEcec9ZJXeva7Y6mj/8egdzwmLtw= github.com/creasty/defaults v1.3.0/go.mod h1:CIEEvs7oIVZm30R8VxtFJs+4k201gReYyuYHJxZc68I= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -112,6 +116,7 @@ github.com/duosecurity/duo_api_golang v0.0.0-20190308151101-6c680f768e74 h1:2MIh github.com/duosecurity/duo_api_golang v0.0.0-20190308151101-6c680f768e74/go.mod h1:UqXY1lYT/ERa4OEAywUqdok1T4RCRdArkhic1Opuavo= github.com/elazarl/go-bindata-assetfs v0.0.0-20160803192304-e1a2a7ec64b0 h1:ZoRgc53qJCfSLimXqJDrmBhnt5GChDsExMCK7t48o0Y= github.com/elazarl/go-bindata-assetfs v0.0.0-20160803192304-e1a2a7ec64b0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.8.0 h1:uE6Fp4fOcAJdc1wTQXLJ+SYistkbG1dNoi6Zs1+Ybvk= github.com/envoyproxy/go-control-plane v0.8.0/go.mod h1:GSSbY9P1neVhdY7G4wu+IK1rk/dqhiCC/4ExuWJZVuk= github.com/envoyproxy/protoc-gen-validate v0.0.14 h1:YBW6/cKy9prEGRYLnaGa4IDhzxZhRCtKsax8srGKDnM= @@ -123,6 +128,7 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/structs v0.0.0-20180123065059-ebf56d35bba7 h1:bGT+Ub6bpzHl7AAYQhBrZ5nYTAH2SF/848WducU0Ao4= github.com/fatih/structs v0.0.0-20180123065059-ebf56d35bba7/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= @@ -136,6 +142,10 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-sql-driver/mysql v0.0.0-20180618115901-749ddf1598b4 h1:1LlmVz15APoKz9dnm5j2ePptburJlwEH+/v/pUuoxck= github.com/go-sql-driver/mysql v0.0.0-20180618115901-749ddf1598b4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -156,6 +166,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= @@ -174,9 +185,11 @@ github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4r github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0= github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.0.0-20180828235145-f29afc2cceca h1:wobTb8SE189AuxzEKClyYxiI4nUGWlpVtl13eLiFlOE= @@ -291,6 +304,7 @@ github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/joyent/triton-go v0.0.0-20180628001255-830d2b111e62 h1:JHCT6xuyPUrbbgAPE/3dqlvUKzRHMNuTBKKUb6OeR/k= github.com/joyent/triton-go v0.0.0-20180628001255-830d2b111e62/go.mod h1:U+RSyWxWd04xTqnuOQxnai7XGS2PrPY2cfGoDKtMHjA= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= @@ -321,6 +335,7 @@ github.com/lib/pq v0.0.0-20180523175426-90697d60dd84 h1:it29sI2IM490luSc3RAhp5Wu github.com/lib/pq v0.0.0-20180523175426-90697d60dd84/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= @@ -350,9 +365,11 @@ github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nacos-group/nacos-sdk-go v0.0.0-20190723125407-0242d42e3dbb h1:lbmvw8r9W55w+aQgWn35W1nuleRIECMoqUrmwAOAvoI= github.com/nacos-group/nacos-sdk-go v0.0.0-20190723125407-0242d42e3dbb/go.mod h1:CEkSvEpoveoYjA81m4HNeYQ0sge0LFGKSEqO3JKHllo= @@ -360,7 +377,9 @@ github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2 h1:BQ1HW github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2/go.mod h1:TLb2Sg7HQcgGdloNxkrmtgDNR9uVYF3lfdFIN4Ro6Sk= github.com/oklog/run v0.0.0-20180308005104-6934b124db28 h1:Hbr3fbVPXea52oPQeP7KLSxP52g6SFaNY1IqAmUyEW0= github.com/oklog/run v0.0.0-20180308005104-6934b124db28/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= @@ -383,6 +402,7 @@ github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= @@ -435,12 +455,14 @@ github.com/softlayer/softlayer-go v0.0.0-20180806151055-260589d94c7d h1:bVQRCxQv github.com/softlayer/softlayer-go v0.0.0-20180806151055-260589d94c7d/go.mod h1:Cw4GTlQccdRGSEf6KiMju767x0NEHE0YIVPJSaXjlsw= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -477,6 +499,7 @@ golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0F golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -496,6 +519,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -508,6 +532,7 @@ golang.org/x/sys v0.0.0-20190508220229-2d0786266e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -517,6 +542,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZe golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -572,3 +598,9 @@ k8s.io/apimachinery v0.0.0-20190223001710-c182ff3b9841 h1:Q4RZrHNtlC/mSdC1sTrcZ5 k8s.io/apimachinery v0.0.0-20190223001710-c182ff3b9841/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/client-go v8.0.0+incompatible h1:tTI4hRmb1DRMl4fG6Vclfdi6nTM82oIrTT7HfitmxC4= k8s.io/client-go v8.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= +k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/registry/kubernetes/listener.go b/registry/kubernetes/listener.go new file mode 100644 index 0000000000..c4c951d754 --- /dev/null +++ b/registry/kubernetes/listener.go @@ -0,0 +1,100 @@ +package kubernetes + +import ( + "context" + "strings" +) + +import ( + perrors "github.com/pkg/errors" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/common/logger" + "github.com/apache/dubbo-go/config_center" + "github.com/apache/dubbo-go/registry" + "github.com/apache/dubbo-go/remoting" +) + +type dataListener struct { + interestedURL []*common.URL + listener config_center.ConfigurationListener +} + +func NewRegistryDataListener(listener config_center.ConfigurationListener) *dataListener { + return &dataListener{listener: listener, interestedURL: []*common.URL{}} +} + +func (l *dataListener) AddInterestedURL(url *common.URL) { + l.interestedURL = append(l.interestedURL, url) +} + +func (l *dataListener) DataChange(eventType remoting.Event) bool { + + index := strings.Index(eventType.Path, "/providers/") + if index == -1 { + logger.Warn("Listen with no url, event.path={%v}", eventType.Path) + return false + } + url := eventType.Path[index+len("/providers/"):] + serviceURL, err := common.NewURL(context.Background(), url) + if err != nil { + logger.Warnf("Listen NewURL(r{%s}) = error{%v}", eventType.Path, err) + return false + } + + for _, v := range l.interestedURL { + if serviceURL.URLEqual(*v) { + l.listener.Process( + &config_center.ConfigChangeEvent{ + Key: eventType.Path, + Value: serviceURL, + ConfigType: eventType.Action, + }, + ) + return true + } + } + + return false +} + +type configurationListener struct { + registry *kubernetesRegistry + events chan *config_center.ConfigChangeEvent +} + +func NewConfigurationListener(reg *kubernetesRegistry) *configurationListener { + // add a new waiter + reg.wg.Add(1) + return &configurationListener{registry: reg, events: make(chan *config_center.ConfigChangeEvent, 32)} +} +func (l *configurationListener) Process(configType *config_center.ConfigChangeEvent) { + l.events <- configType +} + +func (l *configurationListener) Next() (*registry.ServiceEvent, error) { + for { + select { + case <-l.registry.done: + logger.Warnf("listener's kubernetes client connection is broken, so kubernetes event listener exit now.") + return nil, perrors.New("listener stopped") + + case e := <-l.events: + logger.Infof("got kubernetes event %#v", e) + if e.ConfigType == remoting.EventTypeDel { + select { + case <-l.registry.done: + logger.Warnf("update @result{%s}. But its connection to registry is invalid", e.Value) + default: + } + continue + } + return ®istry.ServiceEvent{Action: e.ConfigType, Service: e.Value.(common.URL)}, nil + } + } +} +func (l *configurationListener) Close() { + l.registry.wg.Done() +} diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go new file mode 100644 index 0000000000..276009a41a --- /dev/null +++ b/registry/kubernetes/listener_test.go @@ -0,0 +1 @@ +package kubernetes diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go new file mode 100644 index 0000000000..6c0af8b858 --- /dev/null +++ b/registry/kubernetes/registry.go @@ -0,0 +1,351 @@ +package kubernetes + +import ( + "fmt" + "net/url" + "os" + "path" + "strconv" + "strings" + "sync" + "time" +) + +import ( + gxnet "github.com/dubbogo/gost/net" + perrors "github.com/pkg/errors" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/common/constant" + "github.com/apache/dubbo-go/common/extension" + "github.com/apache/dubbo-go/common/logger" + "github.com/apache/dubbo-go/registry" + "github.com/apache/dubbo-go/remoting/kubernetes" +) + +var ( + processID = "" + localIP = "" +) + +const ( + Name = "kubernetes" + RegistryConnDelay = 3 +) + +func init() { + processID = fmt.Sprintf("%d", os.Getpid()) + localIP, _ = gxnet.GetLocalIP() + extension.SetRegistry(Name, newKubernetesRegistry) +} + +type kubernetesRegistry struct { + *common.URL + birth int64 // time of file birth, seconds since Epoch; 0 if unknown + + cltLock sync.Mutex + client *kubernetes.Client + services map[string]common.URL // service name + protocol -> service config + + listenerLock sync.Mutex + listener *kubernetes.EventListener + dataListener *dataListener + configListener *configurationListener + + wg sync.WaitGroup // wg+done for kubernetes client restart + done chan struct{} +} + +func (r *kubernetesRegistry) Client() *kubernetes.Client { + return r.client +} +func (r *kubernetesRegistry) SetClient(client *kubernetes.Client) { + r.client = client +} +func (r *kubernetesRegistry) ClientLock() *sync.Mutex { + return &r.cltLock +} +func (r *kubernetesRegistry) WaitGroup() *sync.WaitGroup { + return &r.wg +} +func (r *kubernetesRegistry) GetDone() chan struct{} { + return r.done +} +func (r *kubernetesRegistry) RestartCallBack() bool { + + services := []common.URL{} + for _, confIf := range r.services { + services = append(services, confIf) + } + + flag := true + for _, confIf := range services { + err := r.Register(confIf) + if err != nil { + logger.Errorf("(kubernetesProviderRegistry)register(conf{%#v}) = error{%#v}", + confIf, perrors.WithStack(err)) + flag = false + break + } + logger.Infof("success to re-register service :%v", confIf.Key()) + } + return flag +} + +func newKubernetesRegistry(url *common.URL) (registry.Registry, error) { + + r := &kubernetesRegistry{ + URL: url, + birth: time.Now().UnixNano(), + done: make(chan struct{}), + services: make(map[string]common.URL), + } + + if err := kubernetes.ValidateClient(r); err != nil { + return nil, err + } + + r.wg.Add(1) + go kubernetes.HandleClientRestart(r) + + r.listener = kubernetes.NewEventListener(r.client) + r.configListener = NewConfigurationListener(r) + r.dataListener = NewRegistryDataListener(r.configListener) + + return r, nil +} + +func (r *kubernetesRegistry) GetUrl() common.URL { + return *r.URL +} + +func (r *kubernetesRegistry) IsAvailable() bool { + + select { + case <-r.done: + return false + default: + return true + } +} + +func (r *kubernetesRegistry) Destroy() { + + if r.configListener != nil { + r.configListener.Close() + } + r.stop() +} + +func (r *kubernetesRegistry) stop() { + + close(r.done) + + // close current client + r.client.Close() + + r.cltLock.Lock() + r.client = nil + r.services = nil + r.cltLock.Unlock() +} + +func (r *kubernetesRegistry) Register(svc common.URL) error { + + role, err := strconv.Atoi(r.URL.GetParam(constant.ROLE_KEY, "")) + if err != nil { + return perrors.WithMessage(err, "get registry role") + } + + r.cltLock.Lock() + if _, ok := r.services[svc.Key()]; ok { + r.cltLock.Unlock() + return perrors.New(fmt.Sprintf("Path{%s} has been registered", svc.Path)) + } + r.cltLock.Unlock() + + switch role { + case common.PROVIDER: + logger.Debugf("(provider register )Register(conf{%#v})", svc) + if err := r.registerProvider(svc); err != nil { + return perrors.WithMessage(err, "register provider") + } + case common.CONSUMER: + logger.Debugf("(consumer register )Register(conf{%#v})", svc) + if err := r.registerConsumer(svc); err != nil { + return perrors.WithMessage(err, "register consumer") + } + default: + return perrors.New(fmt.Sprintf("unknown role %d", role)) + } + + r.cltLock.Lock() + r.services[svc.Key()] = svc + r.cltLock.Unlock() + return nil +} + +func (r *kubernetesRegistry) createDirIfNotExist(k string) error { + + var tmpPath string + for _, str := range strings.Split(k, "/")[1:] { + tmpPath = path.Join(tmpPath, "/", str) + if err := r.client.Create(tmpPath, ""); err != nil { + return perrors.WithMessagef(err, "create path %s in kubernetes", tmpPath) + } + } + + return nil +} + +func (r *kubernetesRegistry) registerConsumer(svc common.URL) error { + + consumersNode := fmt.Sprintf("/dubbo/%s/%s", svc.Service(), common.DubboNodes[common.CONSUMER]) + if err := r.createDirIfNotExist(consumersNode); err != nil { + logger.Errorf("kubernetes client create path %s: %v", consumersNode, err) + return perrors.WithMessage(err, "kubernetes create consumer nodes") + } + providersNode := fmt.Sprintf("/dubbo/%s/%s", svc.Service(), common.DubboNodes[common.PROVIDER]) + if err := r.createDirIfNotExist(providersNode); err != nil { + return perrors.WithMessage(err, "create provider node") + } + + params := url.Values{} + + params.Add("protocol", svc.Protocol) + + params.Add("category", (common.RoleType(common.CONSUMER)).String()) + params.Add("dubbo", "dubbogo-consumer-"+constant.Version) + + encodedURL := url.QueryEscape(fmt.Sprintf("consumer://%s%s?%s", localIP, svc.Path, params.Encode())) + dubboPath := fmt.Sprintf("/dubbo/%s/%s", svc.Service(), (common.RoleType(common.CONSUMER)).String()) + if err := r.client.Create(path.Join(dubboPath, encodedURL), ""); err != nil { + return perrors.WithMessagef(err, "create k/v in kubernetes (path:%s, url:%s)", dubboPath, encodedURL) + } + + return nil +} + +func (r *kubernetesRegistry) registerProvider(svc common.URL) error { + + if len(svc.Path) == 0 || len(svc.Methods) == 0 { + return perrors.New(fmt.Sprintf("service path %s or service method %s", svc.Path, svc.Methods)) + } + + var ( + urlPath string + encodedURL string + dubboPath string + ) + + providersNode := fmt.Sprintf("/dubbo/%s/%s", svc.Service(), common.DubboNodes[common.PROVIDER]) + if err := r.createDirIfNotExist(providersNode); err != nil { + return perrors.WithMessage(err, "create provider node") + } + + params := url.Values{} + + svc.RangeParams(func(key, value string) bool { + params[key] = []string{value} + return true + }) + params.Add("pid", processID) + params.Add("ip", localIP) + params.Add("anyhost", "true") + params.Add("category", (common.RoleType(common.PROVIDER)).String()) + params.Add("dubbo", "dubbo-provider-golang-"+constant.Version) + params.Add("side", (common.RoleType(common.PROVIDER)).Role()) + + if len(svc.Methods) == 0 { + params.Add("methods", strings.Join(svc.Methods, ",")) + } + + logger.Debugf("provider url params:%#v", params) + var host string + if len(svc.Ip) == 0 { + host = localIP + ":" + svc.Port + } else { + host = svc.Ip + ":" + svc.Port + } + + urlPath = svc.Path + + encodedURL = url.QueryEscape(fmt.Sprintf("%s://%s%s?%s", svc.Protocol, host, urlPath, params.Encode())) + dubboPath = fmt.Sprintf("/dubbo/%s/%s", svc.Service(), (common.RoleType(common.PROVIDER)).String()) + + if err := r.client.Create(path.Join(dubboPath, encodedURL), ""); err != nil { + return perrors.WithMessagef(err, "create k/v in kubernetes (path:%s, url:%s)", dubboPath, encodedURL) + } + + return nil +} + +func (r *kubernetesRegistry) subscribe(svc *common.URL) (registry.Listener, error) { + + var ( + configListener *configurationListener + ) + + r.listenerLock.Lock() + configListener = r.configListener + r.listenerLock.Unlock() + if r.listener == nil { + r.cltLock.Lock() + client := r.client + r.cltLock.Unlock() + if client == nil { + return nil, perrors.New("kubernetes client broken") + } + + // new client & listener + listener := kubernetes.NewEventListener(r.client) + + r.listenerLock.Lock() + r.listener = listener + r.listenerLock.Unlock() + } + + //register the svc to dataListener + r.dataListener.AddInterestedURL(svc) + for _, v := range strings.Split(svc.GetParam(constant.CATEGORY_KEY, constant.DEFAULT_CATEGORY), ",") { + go r.listener.ListenServiceEvent(fmt.Sprintf("/dubbo/%s/"+v, svc.Service()), r.dataListener) + } + + return configListener, nil +} + +//subscribe from registry +func (r *kubernetesRegistry) Subscribe(url *common.URL, notifyListener registry.NotifyListener) { + for { + if !r.IsAvailable() { + logger.Warnf("event listener game over.") + return + } + + listener, err := r.subscribe(url) + if err != nil { + if !r.IsAvailable() { + logger.Warnf("event listener game over.") + return + } + logger.Warnf("getListener() = err:%v", perrors.WithStack(err)) + time.Sleep(time.Duration(RegistryConnDelay) * time.Second) + continue + } + + for { + if serviceEvent, err := listener.Next(); err != nil { + logger.Warnf("Selector.watch() = error{%v}", perrors.WithStack(err)) + listener.Close() + return + } else { + logger.Infof("update begin, service event: %v", serviceEvent.String()) + notifyListener.Notify(serviceEvent) + } + + } + + } +} diff --git a/registry/kubernetes/registry_test.go b/registry/kubernetes/registry_test.go new file mode 100644 index 0000000000..276009a41a --- /dev/null +++ b/registry/kubernetes/registry_test.go @@ -0,0 +1 @@ +package kubernetes diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go new file mode 100644 index 0000000000..e2b0a63884 --- /dev/null +++ b/remoting/kubernetes/client.go @@ -0,0 +1,638 @@ +package kubernetes + +import ( + "context" + "encoding/base64" + "encoding/json" + "os" + "runtime/debug" + "sync" + "time" +) + +import ( + perrors "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +import ( + "github.com/apache/dubbo-go/common/logger" +) + +const ( + // kubernetes inject the var + podNameKey = "HOSTNAME" + nameSpaceKey = "NAMESPACE" + // all pod annotation key + DubboIOAnnotationKey = "dubbo.io/annotation" + + DubboIOLabelKey = "dubbo.io/label" + DubboIOLabelValue = "dubbo.io-value" +) + +var ( + ErrDubboLabelAlreadyExist = perrors.New("dubbo label already exist") + ErrDubboAnnotationsAlreadyExist = perrors.New("dubbo annotations already exist") +) + +type Client struct { + + // kubernetes connection config + cfg *rest.Config + + // the kubernetes interface + rawClient kubernetes.Interface + + // current pod config + currentPodName string + + ns string + + // the memory store + store Store + + // protect the wg && currentPod + lock sync.Mutex + // current pod status + currentPod *v1.Pod + // protect the maintenanceStatus loop && watcher + wg sync.WaitGroup + + // manage the client lifecycle + ctx context.Context + cancel context.CancelFunc +} + +// load CurrentPodName +func getCurrentPodName() (string, error) { + + v := os.Getenv(podNameKey) + if len(v) == 0 { + return "", perrors.New("read value from env by key (HOSTNAME)") + } + return v, nil +} + +// load CurrentNameSpace +func getCurrentNameSpace() (string, error) { + + v := os.Getenv(nameSpaceKey) + if len(v) == 0 { + return "", perrors.New("read value from env by key (NAMESPACE)") + } + return v, nil +} + +// newClient +// new a client for registry +func newClient(namespace string) (*Client, error) { + + cfg, err := rest.InClusterConfig() + if err != nil { + return nil, perrors.WithMessage(err, "get in-cluster config") + } + + rawClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, perrors.WithMessage(err, "new kubernetes client by in cluster config") + } + + currentPodName, err := getCurrentPodName() + if err != nil { + return nil, perrors.WithMessage(err, "get pod name") + } + + ctx, cancel := context.WithCancel(context.Background()) + + c := &Client{ + currentPodName: currentPodName, + ns: namespace, + cfg: cfg, + rawClient: rawClient, + ctx: ctx, + store: newStore(ctx), + cancel: cancel, + } + + currentPod, err := c.initCurrentPod() + if err != nil { + return nil, perrors.WithMessage(err, "init current pod") + } + + // record current status + c.currentPod = currentPod + + // init the store by current pods + if err := c.initStore(); err != nil { + return nil, perrors.WithMessage(err, "init store") + } + + // start kubernetes watch loop + if err := c.maintenanceStatus(); err != nil { + return nil, perrors.WithMessage(err, "maintenance the kubernetes status") + } + + logger.Info("init kubernetes registry success") + return c, nil +} + +// initCurrentPod +// 1. get current pod +// 2. give the dubbo-label for this pod +func (c *Client) initCurrentPod() (*v1.Pod, error) { + + // read the current pod status + currentPod, err := c.rawClient.CoreV1().Pods(c.ns).Get(c.currentPodName, metav1.GetOptions{}) + if err != nil { + return nil, perrors.WithMessagef(err, "get current (%s) pod in namespace (%s)", c.currentPodName, c.ns) + } + + oldPod, newPod, err := c.assembleDUBBOLabel(currentPod) + if err != nil { + if err != ErrDubboLabelAlreadyExist { + return nil, perrors.WithMessage(err, "assemble dubbo label") + } + // current pod don't have label + } + + p, err := c.getPatch(oldPod, newPod) + if err != nil { + return nil, perrors.WithMessage(err, "get patch") + } + + currentPod, err = c.patchCurrentPod(p) + if err != nil { + return nil, perrors.WithMessage(err, "patch to current pod") + } + + return currentPod, nil +} + +// initStore +// 1. get all with dubbo label pods +// 2. put every element to store +func (c *Client) initStore() error { + + pods, err := c.rawClient.CoreV1().Pods(c.ns).List(metav1.ListOptions{ + LabelSelector: fields.OneTermEqualSelector(DubboIOLabelKey, DubboIOLabelValue).String(), + }) + if err != nil { + return perrors.WithMessagef(err, "list pods in namespace (%s)", c.ns) + } + + for _, pod := range pods.Items { + logger.Debugf("got the pod (name: %s), (label: %v), (annotations: %v)", pod.Name, pod.GetLabels(), pod.GetAnnotations()) + c.handleWatchedPodEvent(&pod, watch.Added) + } + + return nil +} + +// maintenanceStatus +// try to watch kubernetes pods +func (c *Client) maintenanceStatus() error { + + c.wg.Add(1) + + // try once + watcher, err := c.rawClient.CoreV1().Pods(c.ns).Watch(metav1.ListOptions{ + LabelSelector: fields.OneTermEqualSelector(DubboIOLabelKey, DubboIOLabelValue).String(), + Watch: true, + }) + if err != nil { + return perrors.WithMessagef(err, "try to watch the namespace (%s) pods", c.ns) + } + + watcher.Stop() + + // add wg, grace close the client + go c.maintenanceStatusLoop() + return nil +} + +// maintenanceStatus +// try to notify +func (c *Client) maintenanceStatusLoop() { + + defer func() { + // notify other goroutine, this loop over + c.wg.Done() + logger.Info("maintenanceStatusLoop goroutine game over") + }() + + var lastResourceVersion string + + for { + + wc, err := c.rawClient.CoreV1().Pods(c.ns).Watch(metav1.ListOptions{ + LabelSelector: fields.OneTermEqualSelector(DubboIOLabelKey, DubboIOLabelValue).String(), + Watch: true, + ResourceVersion: lastResourceVersion, + }) + if err != nil { + logger.Warnf("watch the namespace (%s) pods: %v, retry after 2 seconds", c.ns, err) + time.Sleep(2 * time.Second) + continue + } + + logger.Infof("the old kubernetes client broken, collect the resource status from resource version (%s)", lastResourceVersion) + + select { + case <-c.ctx.Done(): + // the client stopped + logger.Info("the kubernetes client stopped") + return + + default: + + for { + select { + // double check ctx + case <-c.ctx.Done(): + logger.Info("the kubernetes client stopped") + + // get one element from result-chan + case event, ok := <-wc.ResultChan(): + if !ok { + wc.Stop() + logger.Info("kubernetes watch chan die, create new") + goto onceWatch + } + + if event.Type == watch.Error { + // watched a error event + logger.Warnf("kubernetes watch api report err (%#v)", event) + continue + } + + type resourceVersionGetter interface { + GetResourceVersion() string + } + + o, ok := event.Object.(resourceVersionGetter) + if !ok { + continue + } + + // record the last resource version avoid to sync all pod + lastResourceVersion = o.GetResourceVersion() + logger.Infof("kuberentes get the current resource version %v", lastResourceVersion) + + // check event object type + p, ok := event.Object.(*v1.Pod) + if !ok { + // not a pod + continue + } + + // handle the watched pod + go c.handleWatchedPodEvent(p, event.Type) + } + } + onceWatch: + } + } +} + +// handleWatchedPodEvent +// handle watched pod event +func (c *Client) handleWatchedPodEvent(p *v1.Pod, eventType watch.EventType) { + + for ak, av := range p.GetAnnotations() { + + // not dubbo interest annotation + if ak != DubboIOAnnotationKey { + continue + } + + ol, err := c.unmarshalRecord(av) + if err != nil { + logger.Errorf("there a pod with dubbo annotation, but unmarshal dubbo value %v", err) + return + } + + for _, o := range ol { + + switch eventType { + case watch.Added: + // if pod is added, the record always be create + o.EventType = Create + case watch.Modified: + o.EventType = Update + case watch.Deleted: + o.EventType = Delete + default: + logger.Errorf("no valid kubernetes event-type (%s) ", eventType) + return + } + + logger.Debugf("prepare to put object (%#v) to kuberentes-store", o) + + if err := c.store.Put(o); err != nil { + logger.Errorf("put (%#v) to cache store: %v ", o, err) + return + } + + } + + } +} + +// unmarshalRecord +// unmarshal the kubernetes dubbo annotation value +func (c *Client) unmarshalRecord(record string) ([]*Object, error) { + + if len(record) == 0 { + // NOTICE: + // []*Object is nil. + return nil, nil + } + + rawMsg, err := base64.URLEncoding.DecodeString(record) + if err != nil { + return nil, perrors.WithMessagef(err, "decode record (%s)", record) + } + + var out []*Object + if err := json.Unmarshal(rawMsg, &out); err != nil { + return nil, perrors.WithMessage(err, "decode json") + } + return out, nil +} + +// marshalRecord +// marshal the kubernetes dubbo annotation value +func (c *Client) marshalRecord(ol []*Object) (string, error) { + + msg, err := json.Marshal(ol) + if err != nil { + return "", perrors.WithMessage(err, "json encode object list") + } + return base64.URLEncoding.EncodeToString(msg), nil +} + +// readCurrentPod +// read the current pod status from kubernetes api +func (c *Client) readCurrentPod() (*v1.Pod, error) { + + currentPod, err := c.rawClient.CoreV1().Pods(c.ns).Get(c.currentPodName, metav1.GetOptions{}) + if err != nil { + return nil, perrors.WithMessagef(err, "get current (%s) pod in namespace (%s)", c.currentPodName, c.ns) + } + return currentPod, nil +} + +// Create +// create k/v pair in storage +func (c *Client) Create(k, v string) error { + + // 1. accord old pod && (k, v) assemble new pod dubbo annotion v + // 2. get patch data + // 3. PATCH the pod + c.lock.Lock() + defer c.lock.Unlock() + + currentPod, err := c.readCurrentPod() + if err != nil { + return perrors.WithMessage(err, "read current pod") + } + + oldPod, newPod, err := c.assembleDUBBOAnnotations(k, v, currentPod) + if err != nil { + return perrors.WithMessage(err, "assemble") + } + + patchBytes, err := c.getPatch(oldPod, newPod) + if err != nil { + return perrors.WithMessage(err, "get patch") + } + + updatedPod, err := c.patchCurrentPod(patchBytes) + if err != nil { + return perrors.WithMessage(err, "patch current pod") + } + + c.currentPod = updatedPod + // not update the store, the store should be write by the maintenanceStatusLoop + return nil +} + +// patch current pod +// write new meta for current pod +func (c *Client) patchCurrentPod(patch []byte) (*v1.Pod, error) { + + updatedPod, err := c.rawClient.CoreV1().Pods(c.ns).Patch(c.currentPodName, types.StrategicMergePatchType, patch) + if err != nil { + return nil, perrors.WithMessage(err, "patch in kubernetes pod ") + } + return updatedPod, nil +} + +// assemble the dubbo kubernete label +// every dubbo instance should be labeled spec {"dubbo.io/label":"dubbo.io/label-value"} label +func (c *Client) assembleDUBBOLabel(currentPod *v1.Pod) (oldPod *v1.Pod, newPod *v1.Pod, err error) { + + oldPod = &v1.Pod{} + newPod = &v1.Pod{} + + oldPod.Labels = make(map[string]string, 8) + newPod.Labels = make(map[string]string, 8) + + if currentPod.GetLabels() != nil { + + if currentPod.GetLabels()[DubboIOLabelKey] == DubboIOLabelValue { + // already have label + err = ErrDubboLabelAlreadyExist + return + } + } + + // copy current pod labels to oldPod && newPod + for k, v := range currentPod.GetLabels() { + oldPod.Labels[k] = v + newPod.Labels[k] = v + } + // assign new label for current pod + newPod.Labels[DubboIOLabelKey] = DubboIOLabelValue + return +} + +// assemble the dubbo kubernetes annotations +// accord the current pod && (k,v) assemble the old-pod, new-pod +func (c *Client) assembleDUBBOAnnotations(k, v string, currentPod *v1.Pod) (oldPod *v1.Pod, newPod *v1.Pod, err error) { + + oldPod = &v1.Pod{} + newPod = &v1.Pod{} + oldPod.Annotations = make(map[string]string, 8) + newPod.Annotations = make(map[string]string, 8) + + for k, v := range currentPod.GetAnnotations() { + oldPod.Annotations[k] = v + newPod.Annotations[k] = v + } + + al, err := c.unmarshalRecord(oldPod.GetAnnotations()[DubboIOAnnotationKey]) + if err != nil { + err = perrors.WithMessage(err, "unmarshal record") + return + } + + newAnnotations, err := c.marshalRecord(append(al, &Object{Key: k, Value: v})) + if err != nil { + err = perrors.WithMessage(err, "marshal record") + return + } + + newPod.Annotations[DubboIOAnnotationKey] = newAnnotations + return +} + +// getPatch +// get the kubernetes pod patch bytes +func (c *Client) getPatch(oldPod, newPod *v1.Pod) ([]byte, error) { + + oldData, err := json.Marshal(oldPod) + if err != nil { + return nil, perrors.WithMessage(err, "marshal old pod") + } + + newData, err := json.Marshal(newPod) + if err != nil { + return nil, perrors.WithMessage(err, "marshal newPod pod") + } + + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Pod{}) + if err != nil { + return nil, perrors.WithMessage(err, "create two-way-merge-patch") + } + return patchBytes, nil +} + +// GetChildren +// get k children list from kubernetes-store +func (c *Client) GetChildren(k string) ([]string, []string, error) { + + objectList, err := c.store.Get(k, true) + if err != nil { + return nil, nil, perrors.WithMessagef(err, "get children from store on (%s)", k) + } + + var kList []string + var vList []string + + for _, o := range objectList { + kList = append(kList, o.Key) + vList = append(vList, o.Value) + } + + return kList, vList, nil +} + +// Watch +// watch on spec key +func (c *Client) Watch(k string) (<-chan *Object, error) { + + debug.PrintStack() + + w, err := c.store.Watch(k, false) + if err != nil { + return nil, perrors.WithMessagef(err, "watch on (%s)", k) + } + + return w.ResultChan(), nil +} + +// Watch +// watch on spec prefix +func (c *Client) WatchWithPrefix(prefix string) (<-chan *Object, error) { + + w, err := c.store.Watch(prefix, true) + if err != nil { + return nil, perrors.WithMessagef(err, "watch on prefix (%s)", prefix) + } + + return w.ResultChan(), nil +} + +// Valid +// Valid the client +// if return false, the client is die +func (c *Client) Valid() bool { + + select { + case <-c.Done(): + return false + default: + return true + } +} + +// Done +// read the client status +func (c *Client) Done() <-chan struct{} { + return c.ctx.Done() +} + +// Stop +// read the client status +func (c *Client) Close() { + + select { + case <-c.ctx.Done(): + //already stopped + return + default: + } + c.cancel() + + // the client ctx be canceled + // will trigger the store watchers all stopped + // so, just wait + c.wg.Wait() +} + +// ValidateClient +// validate the kubernetes client +func ValidateClient(container clientFacade) error { + + lock := container.ClientLock() + lock.Lock() + defer lock.Unlock() + + // new Client + if container.Client() == nil { + ns, err := getCurrentNameSpace() + if err != nil { + return perrors.WithMessage(err, "get current namespace") + } + newClient, err := newClient(ns) + if err != nil { + logger.Warnf("new kubernetes client (namespace{%s}: %v)", ns, err) + return perrors.WithMessagef(err, "new kubernetes client (:%+v)", ns) + } + container.SetClient(newClient) + } + + if !container.Client().Valid() { + + ns, err := getCurrentNameSpace() + if err != nil { + return perrors.WithMessage(err, "get current namespace") + } + newClient, err := newClient(ns) + if err != nil { + logger.Warnf("new kubernetes client (namespace{%s}: %v)", ns, err) + return perrors.WithMessagef(err, "new kubernetes client (:%+v)", ns) + } + container.SetClient(newClient) + } + + return nil +} diff --git a/remoting/kubernetes/facade.go b/remoting/kubernetes/facade.go new file mode 100644 index 0000000000..4a64150575 --- /dev/null +++ b/remoting/kubernetes/facade.go @@ -0,0 +1,77 @@ +package kubernetes + +import ( + "sync" +) + +import ( + "github.com/dubbogo/getty" + perrors "github.com/pkg/errors" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/common/logger" +) + +const ( + ConnDelay = 3 + MaxFailTimes = 15 +) + +type clientFacade interface { + Client() *Client + SetClient(*Client) + ClientLock() *sync.Mutex + WaitGroup() *sync.WaitGroup + GetDone() chan struct{} + RestartCallBack() bool + common.Node +} + +func HandleClientRestart(r clientFacade) { + + var ( + err error + failTimes int + ) + + defer r.WaitGroup().Done() +LOOP: + for { + select { + case <-r.GetDone(): + logger.Warnf("(KubernetesProviderRegistry)reconnectKubernetes goroutine exit now...") + break LOOP + // re-register all services + case <-r.Client().Done(): + r.ClientLock().Lock() + r.Client().Close() + r.SetClient(nil) + r.ClientLock().Unlock() + + // try to connect to kubernetes, + failTimes = 0 + for { + select { + case <-r.GetDone(): + logger.Warnf("(KubernetesProviderRegistry)reconnectKubernetes Registry goroutine exit now...") + break LOOP + case <-getty.GetTimeWheel().After(timeSecondDuration(failTimes * ConnDelay)): // avoid connect frequent + } + err = ValidateClient(r) + logger.Infof("Kubernetes ProviderRegistry.validateKubernetesClient = error{%#v}", perrors.WithStack(err)) + + if err == nil { + if r.RestartCallBack() { + break + } + } + failTimes++ + if MaxFailTimes <= failTimes { + failTimes = MaxFailTimes + } + } + } + } +} diff --git a/remoting/kubernetes/listener.go b/remoting/kubernetes/listener.go new file mode 100644 index 0000000000..48e1cc160a --- /dev/null +++ b/remoting/kubernetes/listener.go @@ -0,0 +1,188 @@ +package kubernetes + +import ( + "sync" + "time" +) + +import ( + perrors "github.com/pkg/errors" +) + +import ( + "github.com/apache/dubbo-go/common/logger" + "github.com/apache/dubbo-go/remoting" +) + +type EventListener struct { + client *Client + keyMapLock sync.Mutex + keyMap map[string]struct{} + wg sync.WaitGroup +} + +func NewEventListener(client *Client) *EventListener { + return &EventListener{ + client: client, + keyMap: make(map[string]struct{}), + } +} + +// Listen on a spec key +// this method will return true when spec key deleted, +// this method will return false when deep layer connection lose +func (l *EventListener) ListenServiceNodeEvent(key string, listener ...remoting.DataListener) bool { + l.wg.Add(1) + defer l.wg.Done() + for { + wc, err := l.client.Watch(key) + if err != nil { + logger.Warnf("watch exist{key:%s} = error{%v}", key, err) + return false + } + + select { + + // client stopped + case <-l.client.Done(): + logger.Warnf("kubernetes client stopped") + return false + + // handle kubernetes-store events + case e, ok := <-wc: + if !ok { + logger.Warnf("kubernetes-store watch-chan closed") + return false + } + + if l.handleEvents(e, listener...) { + // if event is delete + return true + } + } + } + + return false +} + +// return true mean the event type is DELETE +// return false mean the event type is CREATE || UPDATE +func (l *EventListener) handleEvents(event *Object, listeners ...remoting.DataListener) bool { + + logger.Infof("got a kubernetes-store event {type: %d, key: %s}", event.EventType, event.Key) + + switch event.EventType { + case Create: + for _, listener := range listeners { + logger.Infof("kubernetes-store get event (key{%s}) = event{EventNodeDataCreated}", event.Key) + listener.DataChange(remoting.Event{ + Path: string(event.Key), + Action: remoting.EventTypeAdd, + Content: string(event.Value), + }) + } + return false + case Update: + for _, listener := range listeners { + logger.Infof("kubernetes-store get event (key{%s}) = event{EventNodeDataChanged}", event.Key) + listener.DataChange(remoting.Event{ + Path: string(event.Key), + Action: remoting.EventTypeUpdate, + Content: string(event.Value), + }) + } + return false + case Delete: + logger.Warnf("kubernetes-store get event (key{%s}) = event{EventNodeDeleted}", event.Key) + return true + default: + return false + } +} + +// Listen on a set of key with spec prefix +func (l *EventListener) ListenServiceNodeEventWithPrefix(prefix string, listener ...remoting.DataListener) { + + l.wg.Add(1) + defer l.wg.Done() + for { + wc, err := l.client.WatchWithPrefix(prefix) + if err != nil { + logger.Warnf("listenDirEvent(key{%s}) = error{%v}", prefix, err) + } + + select { + // client stopped + case <-l.client.Done(): + logger.Warnf("kubernetes client stopped") + return + + // kuberentes-store event stream + case e, ok := <-wc: + + if !ok { + logger.Warnf("kubernetes-store watch-chan closed") + return + } + + l.handleEvents(e, listener...) + } + } +} + +func timeSecondDuration(sec int) time.Duration { + return time.Duration(sec) * time.Second +} + +// this func is invoked by kubernetes ConsumerRegistry::Registry/ kubernetes ConsumerRegistry::get/kubernetes ConsumerRegistry::getListener +// registry.go:Listen -> listenServiceEvent -> listenDirEvent -> ListenServiceNodeEvent +// | +// --------> ListenServiceNodeEvent +func (l *EventListener) ListenServiceEvent(key string, listener remoting.DataListener) { + + l.keyMapLock.Lock() + _, ok := l.keyMap[key] + l.keyMapLock.Unlock() + if ok { + logger.Warnf("kubernetes-store key %s has already been listened.", key) + return + } + + l.keyMapLock.Lock() + l.keyMap[key] = struct{}{} + l.keyMapLock.Unlock() + + keyList, valueList, err := l.client.GetChildren(key) + if err != nil { + logger.Errorf("Get new node path {%v} 's content error,message is {%v}", key, perrors.WithMessage(err, "get children")) + } + + logger.Infof("get key children list %s, keys %v values %v", key, keyList, valueList) + + for i, k := range keyList { + logger.Infof("got children list key -> %s", k) + listener.DataChange(remoting.Event{ + Path: k, + Action: remoting.EventTypeAdd, + Content: valueList[i], + }) + } + + logger.Infof("listen dubbo provider key{%s} event and wait to get all provider from kubernetes-store", key) + go func(key string, listener remoting.DataListener) { + l.ListenServiceNodeEventWithPrefix(key, listener) + logger.Warnf("listenDirEvent(key{%s}) goroutine exit now", key) + }(key, listener) + + logger.Infof("listen dubbo service key{%s}", key) + go func(key string) { + if l.ListenServiceNodeEvent(key) { + listener.DataChange(remoting.Event{Path: key, Action: remoting.EventTypeDel}) + } + logger.Warnf("listenSelf(kubernetes key{%s}) goroutine exit now", key) + }(key) +} + +func (l *EventListener) Close() { + l.wg.Wait() +} diff --git a/remoting/kubernetes/store.go b/remoting/kubernetes/store.go new file mode 100644 index 0000000000..f58291d0e2 --- /dev/null +++ b/remoting/kubernetes/store.go @@ -0,0 +1,305 @@ +package kubernetes + +import ( + "context" + "strconv" + "strings" + "sync" +) + +import ( + perrors "github.com/pkg/errors" +) + +var ( + ErrStoreAlreadyStopped = perrors.New("the store already be stopped") + ErrKVPairNotFound = perrors.New("k/v pair not found") +) + +const ( + defaultWatcherChanSize = 100 +) + +type eventType int + +const ( + Create eventType = iota + Update + Delete +) + +func (e eventType) String() string { + + switch e { + case Create: + return "CREATE" + case Update: + return "UPDATE" + case Delete: + return "DELETE" + default: + return "UNKNOWN" + } +} + +// Object +// object is element in store +type Object struct { + // event-type + EventType eventType `json:"-"` + // the dubbo-go should consume the key + Key string `json:"k"` + // the dubbo-go should consume the value + Value string `json:"v"` +} + +// Watchable Store +type Store interface { + + // put the object to the store + Put(object *Object) error + // if prefix is false, + // the len([]*Object) == 0 + Get(key string, prefix bool) ([]*Object, error) + // watch the spec key or key prefix + Watch(key string, prefix bool) (Watcher, error) + // check the store status + Done() <-chan struct{} +} + +// Stopped Watcher +type Watcher interface { + // the watcher's id + ID() string + // result stream + ResultChan() <-chan *Object + // Stop the watcher + stop() + // check the watcher status + done() <-chan struct{} +} + +// the store +type storeImpl struct { + + // Client's ctx, client die, the store will die too + ctx context.Context + + // protect store and watchers + lock sync.RWMutex + + // the key is dubbo-go interest meta + cache map[string]*Object + + currentWatcherId uint64 + watchers map[uint64]*watcher +} + +func (s *storeImpl) loop() { + + select { + case <-s.ctx.Done(): + // parent ctx be canceled, close the store + s.lock.Lock() + defer s.lock.Unlock() + for _, w := range s.watchers { + // stop data stream + close(w.ch) + // stop watcher + w.stop() + } + } +} + +// Watch +// watch on spec key, with or without prefix +func (s *storeImpl) Watch(key string, prefix bool) (Watcher, error) { + return s.addWatcher(key, prefix) +} + +// Done +// get the store status +func (s *storeImpl) Done() <-chan struct{} { + return s.ctx.Done() +} + +// Put +// put the object to store +func (s *storeImpl) Put(object *Object) error { + + sendMsg := func(object *Object, w *watcher) { + s.lock.Lock() + defer s.lock.Unlock() + select { + case <-w.done(): + // the watcher already stop + case w.ch <- object: + // block send the msg + } + } + + s.lock.Lock() + defer s.lock.Unlock() + + if err := s.valid(); err != nil { + return err + } + + // put to store + if object.EventType == Delete { + delete(s.cache, object.Key) + } else { + s.cache[object.Key] = object + } + + // notify watcher + for _, w := range s.watchers { + + if !strings.Contains(object.Key, w.interested.key) { + // this watcher no interest in this element + continue + } + + if !w.interested.prefix { + if object.Key == w.interested.key { + go sendMsg(object, w) + } + // not interest + continue + } + go sendMsg(object, w) + } + return nil +} + +// valid +// valid the client status +// NOTICE: +// should protected by lock +func (s *storeImpl) valid() error { + select { + case <-s.ctx.Done(): + return ErrStoreAlreadyStopped + default: + return nil + } +} + +// addWatcher +func (s *storeImpl) addWatcher(key string, prefix bool) (Watcher, error) { + + w := &watcher{ + store: s, + interested: struct { + key string + prefix bool + }{key: key, prefix: prefix}, + ch: make(chan *Object, defaultWatcherChanSize), + exit: make(chan struct{}), + } + + s.lock.Lock() + defer s.lock.Unlock() + + if err := s.valid(); err != nil { + return nil, err + } + + s.watchers[s.currentWatcherId] = w + w.id = s.currentWatcherId + s.currentWatcherId = s.currentWatcherId + 1 + return w, nil +} + +// Get +// get elements from cache +func (s *storeImpl) Get(key string, prefix bool) ([]*Object, error) { + + s.lock.RLock() + defer s.lock.RUnlock() + + if err := s.valid(); err != nil { + return nil, err + } + + if !prefix { + for k, v := range s.cache { + if k == key { + return []*Object{v}, nil + } + } + // object + return nil, ErrKVPairNotFound + } + + var out []*Object + + for k, v := range s.cache { + if strings.Contains(k, key) { + out = append(out, v) + } + } + + if len(out) == 0 { + return nil, ErrKVPairNotFound + } + + return out, nil +} + +// the store watcher +type watcher struct { + id uint64 + + // the underlay store + store *storeImpl + + // the interest topic + interested struct { + key string + prefix bool + } + ch chan *Object + + closeOnce sync.Once + exit chan struct{} +} + +// ResultChan +func (w *watcher) ResultChan() <-chan *Object { + return w.ch +} + +// ID +// the watcher's id +func (w *watcher) ID() string { + return strconv.FormatUint(w.id, 10) +} + +// stop +// stop the watcher +func (w *watcher) stop() { + + // double close will panic + w.closeOnce.Do(func() { + close(w.exit) + }) +} + +// done +// check watcher status +func (w *watcher) done() <-chan struct{} { + return w.exit +} + +// newStore +// new store from parent context +func newStore(ctx context.Context) Store { + s := &storeImpl{ + ctx: ctx, + cache: map[string]*Object{}, + watchers: map[uint64]*watcher{}, + } + go s.loop() + return s +} diff --git a/remoting/kubernetes/store_test.go b/remoting/kubernetes/store_test.go new file mode 100644 index 0000000000..5cb844f370 --- /dev/null +++ b/remoting/kubernetes/store_test.go @@ -0,0 +1,66 @@ +package kubernetes + +import ( + "context" + "fmt" + "strconv" + "sync" + "testing" + "time" +) + +func TestStore(t *testing.T) { + + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + defer cancel() + + s := newStore(ctx) + + wg := sync.WaitGroup{} + + for i := 0; i < 2; i++ { + + wg.Add(1) + + go func() { + defer wg.Done() + w, err := s.Watch("key-1", false) + if err != nil { + fmt.Println("watch spec result", err) + return + } + for e := range w.ResultChan() { + fmt.Printf("consumer %s got %s\n", w.ID(), e.Key) + } + }() + } + for i := 2; i < 3; i++ { + + wg.Add(1) + go func() { + + defer wg.Done() + w, err := s.Watch("key", true) + if err != nil { + fmt.Println("watch prefix result", err) + return + } + for e := range w.ResultChan() { + fmt.Printf("prefix consumer %s got %s\n", w.ID(), e.Key) + } + }() + } + + for i := 0; i < 5; i++ { + go func(i int) { + if err := s.Put(&Object{ + Key: "key-" + strconv.Itoa(i), + Value: strconv.Itoa(i), + }); err != nil { + t.Fatal(err) + } + }(i) + } + + wg.Wait() +} From 32e57582494d4cecb6ab987a292798dcc57a9ea1 Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 13 Jan 2020 11:23:20 +0800 Subject: [PATCH 05/61] Fix the kubernetes && etcd registry race-condition --- registry/etcdv3/registry.go | 7 ++++++- registry/kubernetes/registry.go | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/registry/etcdv3/registry.go b/registry/etcdv3/registry.go index b058113c69..db5d783b4d 100644 --- a/registry/etcdv3/registry.go +++ b/registry/etcdv3/registry.go @@ -334,7 +334,12 @@ func (r *etcdV3Registry) subscribe(svc *common.URL) (registry.Listener, error) { listener := etcdv3.NewEventListener(r.client) r.listenerLock.Lock() - r.listener = listener + // NOTICE: + // double-check the listener + // if r.listener already be assigned, discard the new value + if r.listener == nil{ + r.listener = listener + } r.listenerLock.Unlock() } diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index 6c0af8b858..d7470cbb9f 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -303,7 +303,12 @@ func (r *kubernetesRegistry) subscribe(svc *common.URL) (registry.Listener, erro listener := kubernetes.NewEventListener(r.client) r.listenerLock.Lock() - r.listener = listener + // NOTICE: + // double-check the listener + // if r.listener already be assigned, discard the new value + if r.listener == nil{ + r.listener = listener + } r.listenerLock.Unlock() } From 723aa683fdf43f3a10284b8a1952653c3cfa267d Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 13 Jan 2020 11:26:19 +0800 Subject: [PATCH 06/61] Fix bool value return --- registry/etcdv3/registry.go | 6 ++---- registry/kubernetes/registry.go | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/registry/etcdv3/registry.go b/registry/etcdv3/registry.go index db5d783b4d..e328c5c730 100644 --- a/registry/etcdv3/registry.go +++ b/registry/etcdv3/registry.go @@ -97,18 +97,16 @@ func (r *etcdV3Registry) RestartCallBack() bool { services = append(services, confIf) } - flag := true for _, confIf := range services { err := r.Register(confIf) if err != nil { logger.Errorf("(etcdV3ProviderRegistry)register(conf{%#v}) = error{%#v}", confIf, perrors.WithStack(err)) - flag = false - break + return false } logger.Infof("success to re-register service :%v", confIf.Key()) } - return flag + return true } func newETCDV3Registry(url *common.URL) (registry.Registry, error) { diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index d7470cbb9f..47a9ea8814 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -80,18 +80,16 @@ func (r *kubernetesRegistry) RestartCallBack() bool { services = append(services, confIf) } - flag := true for _, confIf := range services { err := r.Register(confIf) if err != nil { logger.Errorf("(kubernetesProviderRegistry)register(conf{%#v}) = error{%#v}", confIf, perrors.WithStack(err)) - flag = false - break + return false } logger.Infof("success to re-register service :%v", confIf.Key()) } - return flag + return true } func newKubernetesRegistry(url *common.URL) (registry.Registry, error) { From 2ec84e5d78faa886465c355729cbe7993870f32f Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 13 Jan 2020 11:32:02 +0800 Subject: [PATCH 07/61] Delete the unused check block --- registry/etcdv3/registry.go | 4 ---- registry/kubernetes/registry.go | 4 ---- 2 files changed, 8 deletions(-) diff --git a/registry/etcdv3/registry.go b/registry/etcdv3/registry.go index e328c5c730..64eca0dc36 100644 --- a/registry/etcdv3/registry.go +++ b/registry/etcdv3/registry.go @@ -287,10 +287,6 @@ func (r *etcdV3Registry) registerProvider(svc common.URL) error { params.Add("dubbo", "dubbo-provider-golang-"+constant.Version) params.Add("side", (common.RoleType(common.PROVIDER)).Role()) - if len(svc.Methods) == 0 { - params.Add("methods", strings.Join(svc.Methods, ",")) - } - logger.Debugf("provider url params:%#v", params) var host string if len(svc.Ip) == 0 { diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index 47a9ea8814..6f05b107b6 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -256,10 +256,6 @@ func (r *kubernetesRegistry) registerProvider(svc common.URL) error { params.Add("dubbo", "dubbo-provider-golang-"+constant.Version) params.Add("side", (common.RoleType(common.PROVIDER)).Role()) - if len(svc.Methods) == 0 { - params.Add("methods", strings.Join(svc.Methods, ",")) - } - logger.Debugf("provider url params:%#v", params) var host string if len(svc.Ip) == 0 { From daac5e123d84f64134808da08859664d4baf7fad Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 13 Jan 2020 11:33:43 +0800 Subject: [PATCH 08/61] Add apache license --- registry/etcdv3/registry.go | 2 +- registry/kubernetes/listener.go | 17 +++++++++++++++++ registry/kubernetes/listener_test.go | 17 +++++++++++++++++ registry/kubernetes/registry.go | 19 ++++++++++++++++++- registry/kubernetes/registry_test.go | 17 +++++++++++++++++ remoting/kubernetes/client.go | 17 +++++++++++++++++ remoting/kubernetes/facade.go | 17 +++++++++++++++++ remoting/kubernetes/listener.go | 17 +++++++++++++++++ remoting/kubernetes/store.go | 17 +++++++++++++++++ remoting/kubernetes/store_test.go | 17 +++++++++++++++++ 10 files changed, 155 insertions(+), 2 deletions(-) diff --git a/registry/etcdv3/registry.go b/registry/etcdv3/registry.go index 64eca0dc36..68ad456ca1 100644 --- a/registry/etcdv3/registry.go +++ b/registry/etcdv3/registry.go @@ -331,7 +331,7 @@ func (r *etcdV3Registry) subscribe(svc *common.URL) (registry.Listener, error) { // NOTICE: // double-check the listener // if r.listener already be assigned, discard the new value - if r.listener == nil{ + if r.listener == nil { r.listener = listener } r.listenerLock.Unlock() diff --git a/registry/kubernetes/listener.go b/registry/kubernetes/listener.go index c4c951d754..0a494a422d 100644 --- a/registry/kubernetes/listener.go +++ b/registry/kubernetes/listener.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package kubernetes import ( diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go index 276009a41a..7a3ea6a954 100644 --- a/registry/kubernetes/listener_test.go +++ b/registry/kubernetes/listener_test.go @@ -1 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package kubernetes diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index 6f05b107b6..81701dea0b 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package kubernetes import ( @@ -300,7 +317,7 @@ func (r *kubernetesRegistry) subscribe(svc *common.URL) (registry.Listener, erro // NOTICE: // double-check the listener // if r.listener already be assigned, discard the new value - if r.listener == nil{ + if r.listener == nil { r.listener = listener } r.listenerLock.Unlock() diff --git a/registry/kubernetes/registry_test.go b/registry/kubernetes/registry_test.go index 276009a41a..7a3ea6a954 100644 --- a/registry/kubernetes/registry_test.go +++ b/registry/kubernetes/registry_test.go @@ -1 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package kubernetes diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index e2b0a63884..a81c85abd0 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package kubernetes import ( diff --git a/remoting/kubernetes/facade.go b/remoting/kubernetes/facade.go index 4a64150575..0ab689d517 100644 --- a/remoting/kubernetes/facade.go +++ b/remoting/kubernetes/facade.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package kubernetes import ( diff --git a/remoting/kubernetes/listener.go b/remoting/kubernetes/listener.go index 48e1cc160a..5c23d7872f 100644 --- a/remoting/kubernetes/listener.go +++ b/remoting/kubernetes/listener.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package kubernetes import ( diff --git a/remoting/kubernetes/store.go b/remoting/kubernetes/store.go index f58291d0e2..d7f35ced63 100644 --- a/remoting/kubernetes/store.go +++ b/remoting/kubernetes/store.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package kubernetes import ( diff --git a/remoting/kubernetes/store_test.go b/remoting/kubernetes/store_test.go index 5cb844f370..e4acee4b38 100644 --- a/remoting/kubernetes/store_test.go +++ b/remoting/kubernetes/store_test.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package kubernetes import ( From 347124da2ae5b336905c3c7adc49fafd393391dc Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 13 Jan 2020 11:41:04 +0800 Subject: [PATCH 09/61] Fix test embed etcd-server workdir --- registry/etcdv3/listener_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/registry/etcdv3/listener_test.go b/registry/etcdv3/listener_test.go index c064f99c6c..37b03f7102 100644 --- a/registry/etcdv3/listener_test.go +++ b/registry/etcdv3/listener_test.go @@ -19,7 +19,6 @@ package etcdv3 import ( "context" - "github.com/apache/dubbo-go/config_center" "testing" "time" ) @@ -32,6 +31,7 @@ import ( import ( "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/config_center" "github.com/apache/dubbo-go/remoting" ) @@ -46,7 +46,8 @@ func (suite *RegistryTestSuite) SetupSuite() { t := suite.T() cfg := embed.NewConfig() - cfg.Dir = "/tmp/default.etcd" + // avoid conflict with default etcd work-dir + cfg.Dir = "/tmp/default-dubbo-go.etcd" e, err := embed.StartEtcd(cfg) if err != nil { t.Fatal(err) From 460fed6523b6263ed2bafc25fdf79c65bb1c813f Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 13 Jan 2020 11:49:56 +0800 Subject: [PATCH 10/61] Delete the etcd test-server workdir after ut --- registry/etcdv3/listener_test.go | 5 +++++ registry/kubernetes/listener_test.go | 30 ++++++++++++++++++++++++++++ registry/kubernetes/registry_test.go | 18 +++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/registry/etcdv3/listener_test.go b/registry/etcdv3/listener_test.go index 37b03f7102..06610c9591 100644 --- a/registry/etcdv3/listener_test.go +++ b/registry/etcdv3/listener_test.go @@ -19,6 +19,7 @@ package etcdv3 import ( "context" + "os/exec" "testing" "time" ) @@ -67,6 +68,10 @@ func (suite *RegistryTestSuite) SetupSuite() { // stop etcd server func (suite *RegistryTestSuite) TearDownSuite() { suite.etcd.Close() + // clean the etcd workdir + if err := exec.Command("rm", "-rf","/tmp/default-dubbo-go.etcd").Run(); err != nil{ + suite.FailNow(err.Error()) + } } func (suite *RegistryTestSuite) TestDataChange() { diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go index 7a3ea6a954..035fe34e05 100644 --- a/registry/kubernetes/listener_test.go +++ b/registry/kubernetes/listener_test.go @@ -16,3 +16,33 @@ */ package kubernetes + +import ( + "context" + "testing" +) + +import ( + "github.com/stretchr/testify/assert" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/config_center" + "github.com/apache/dubbo-go/remoting" +) + +func Test_DataChange(t *testing.T) { + listener := NewRegistryDataListener(&MockDataListener{}) + url, _ := common.NewURL(context.TODO(), "jsonrpc%3A%2F%2F127.0.0.1%3A20001%2Fcom.ikurento.user.UserProvider%3Fanyhost%3Dtrue%26app.version%3D0.0.1%26application%3DBDTService%26category%3Dproviders%26cluster%3Dfailover%26dubbo%3Ddubbo-provider-golang-2.6.0%26environment%3Ddev%26group%3D%26interface%3Dcom.ikurento.user.UserProvider%26ip%3D10.32.20.124%26loadbalance%3Drandom%26methods.GetUser.loadbalance%3Drandom%26methods.GetUser.retries%3D1%26methods.GetUser.weight%3D0%26module%3Ddubbogo%2Buser-info%2Bserver%26name%3DBDTService%26organization%3Dikurento.com%26owner%3DZX%26pid%3D74500%26retries%3D0%26service.filter%3Decho%26side%3Dprovider%26timestamp%3D1560155407%26version%3D%26warmup%3D100") + listener.AddInterestedURL(&url) + int := listener.DataChange(remoting.Event{Path: "/dubbo/com.ikurento.user.UserProvider/providers/jsonrpc%3A%2F%2F127.0.0.1%3A20001%2Fcom.ikurento.user.UserProvider%3Fanyhost%3Dtrue%26app.version%3D0.0.1%26application%3DBDTService%26category%3Dproviders%26cluster%3Dfailover%26dubbo%3Ddubbo-provider-golang-2.6.0%26environment%3Ddev%26group%3D%26interface%3Dcom.ikurento.user.UserProvider%26ip%3D10.32.20.124%26loadbalance%3Drandom%26methods.GetUser.loadbalance%3Drandom%26methods.GetUser.retries%3D1%26methods.GetUser.weight%3D0%26module%3Ddubbogo%2Buser-info%2Bserver%26name%3DBDTService%26organization%3Dikurento.com%26owner%3DZX%26pid%3D74500%26retries%3D0%26service.filter%3Decho%26side%3Dprovider%26timestamp%3D1560155407%26version%3D%26warmup%3D100"}) + assert.Equal(t, true, int) +} + +type MockDataListener struct { +} + +func (*MockDataListener) Process(configType *config_center.ConfigChangeEvent) { +} + diff --git a/registry/kubernetes/registry_test.go b/registry/kubernetes/registry_test.go index 7a3ea6a954..134217f3c8 100644 --- a/registry/kubernetes/registry_test.go +++ b/registry/kubernetes/registry_test.go @@ -16,3 +16,21 @@ */ package kubernetes + +import ( + "testing" +) + +func Test_Register(t *testing.T) { + +} + +func Test_Subscribe(t *testing.T) { +} + +func Test_ConsumerDestory(t *testing.T) { + +} + +func Test_ProviderDestory(t *testing.T) { +} From 156bf40fa126661509bb79cef393eb6cd8e1c6dc Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 13 Jan 2020 12:02:21 +0800 Subject: [PATCH 11/61] Fix etcd work-dir conflict --- registry/etcdv3/listener_test.go | 4 +-- registry/kubernetes/listener_test.go | 1 - remoting/etcdv3/client_test.go | 6 ++++- remoting/kubernetes/listener_test.go | 38 ++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 remoting/kubernetes/listener_test.go diff --git a/registry/etcdv3/listener_test.go b/registry/etcdv3/listener_test.go index 06610c9591..5a2a6ffeff 100644 --- a/registry/etcdv3/listener_test.go +++ b/registry/etcdv3/listener_test.go @@ -48,7 +48,7 @@ func (suite *RegistryTestSuite) SetupSuite() { cfg := embed.NewConfig() // avoid conflict with default etcd work-dir - cfg.Dir = "/tmp/default-dubbo-go.etcd" + cfg.Dir = "/tmp/default-dubbo-go-registry.etcd" e, err := embed.StartEtcd(cfg) if err != nil { t.Fatal(err) @@ -69,7 +69,7 @@ func (suite *RegistryTestSuite) SetupSuite() { func (suite *RegistryTestSuite) TearDownSuite() { suite.etcd.Close() // clean the etcd workdir - if err := exec.Command("rm", "-rf","/tmp/default-dubbo-go.etcd").Run(); err != nil{ + if err := exec.Command("rm", "-rf", "/tmp/default-dubbo-go-registry.etcd").Run(); err != nil { suite.FailNow(err.Error()) } } diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go index 035fe34e05..866f594d31 100644 --- a/registry/kubernetes/listener_test.go +++ b/registry/kubernetes/listener_test.go @@ -45,4 +45,3 @@ type MockDataListener struct { func (*MockDataListener) Process(configType *config_center.ConfigChangeEvent) { } - diff --git a/remoting/etcdv3/client_test.go b/remoting/etcdv3/client_test.go index 8f9b80cd30..e9aea31bc8 100644 --- a/remoting/etcdv3/client_test.go +++ b/remoting/etcdv3/client_test.go @@ -20,6 +20,7 @@ package etcdv3 import ( "fmt" "net/url" + "os/exec" "path" "reflect" "strings" @@ -91,7 +92,7 @@ func (suite *ClientTestSuite) SetupSuite() { cfg := embed.NewConfig() cfg.LPUrls = []url.URL{*lpurl} cfg.LCUrls = []url.URL{*lcurl} - cfg.Dir = "/tmp/default.etcd" + cfg.Dir = "/tmp/default-dubbo-go-remote.etcd" e, err := embed.StartEtcd(cfg) if err != nil { t.Fatal(err) @@ -111,6 +112,9 @@ func (suite *ClientTestSuite) SetupSuite() { // stop etcd server func (suite *ClientTestSuite) TearDownSuite() { suite.etcd.Close() + if err := exec.Command("rm", "-rf", "/tmp/default-dubbo-go-remote.etcd").Run(); err != nil { + suite.FailNow(err.Error()) + } } func (suite *ClientTestSuite) setUpClient() *Client { diff --git a/remoting/kubernetes/listener_test.go b/remoting/kubernetes/listener_test.go new file mode 100644 index 0000000000..2b6883a076 --- /dev/null +++ b/remoting/kubernetes/listener_test.go @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kubernetes + +import ( + "github.com/apache/dubbo-go/remoting" +) + +type mockDataListener struct { + eventList []remoting.Event + client *Client + changedData string + + rc chan remoting.Event +} + +func (m *mockDataListener) DataChange(eventType remoting.Event) bool { + m.eventList = append(m.eventList, eventType) + if eventType.Content == m.changedData { + m.rc <- eventType + } + return true +} From 04ee311974def10e182829924c4e65e4acc2eeeb Mon Sep 17 00:00:00 2001 From: sxllwx Date: Sat, 18 Jan 2020 12:32:10 +0800 Subject: [PATCH 12/61] fix latest issue --- registry/etcdv3/listener_test.go | 6 ++++-- registry/kubernetes/registry.go | 10 ++++++---- remoting/etcdv3/client_test.go | 6 ++++-- remoting/kubernetes/client.go | 3 +-- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/registry/etcdv3/listener_test.go b/registry/etcdv3/listener_test.go index 5a2a6ffeff..842a216b94 100644 --- a/registry/etcdv3/listener_test.go +++ b/registry/etcdv3/listener_test.go @@ -41,6 +41,8 @@ type RegistryTestSuite struct { etcd *embed.Etcd } +const defaultEtcdV3WorkDir = "/tmp/default-dubbo-go-registry.etcd" + // start etcd server func (suite *RegistryTestSuite) SetupSuite() { @@ -48,7 +50,7 @@ func (suite *RegistryTestSuite) SetupSuite() { cfg := embed.NewConfig() // avoid conflict with default etcd work-dir - cfg.Dir = "/tmp/default-dubbo-go-registry.etcd" + cfg.Dir = defaultEtcdV3WorkDir e, err := embed.StartEtcd(cfg) if err != nil { t.Fatal(err) @@ -69,7 +71,7 @@ func (suite *RegistryTestSuite) SetupSuite() { func (suite *RegistryTestSuite) TearDownSuite() { suite.etcd.Close() // clean the etcd workdir - if err := exec.Command("rm", "-rf", "/tmp/default-dubbo-go-registry.etcd").Run(); err != nil { + if err := exec.Command("rm", "-rf", defaultEtcdV3WorkDir).Run(); err != nil { suite.FailNow(err.Error()) } } diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index 81701dea0b..1153078dc4 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -222,10 +222,12 @@ func (r *kubernetesRegistry) registerConsumer(svc common.URL) error { logger.Errorf("kubernetes client create path %s: %v", consumersNode, err) return perrors.WithMessage(err, "kubernetes create consumer nodes") } - providersNode := fmt.Sprintf("/dubbo/%s/%s", svc.Service(), common.DubboNodes[common.PROVIDER]) - if err := r.createDirIfNotExist(providersNode); err != nil { - return perrors.WithMessage(err, "create provider node") - } + + // NOTICE kubernetes && etcdv3 not need create provider metadata dir in consumer logic + //providersNode := fmt.Sprintf("/dubbo/%s/%s", svc.Service(), common.DubboNodes[common.PROVIDER]) + //if err := r.createDirIfNotExist(providersNode); err != nil { + // return perrors.WithMessage(err, "create provider node") + //} params := url.Values{} diff --git a/remoting/etcdv3/client_test.go b/remoting/etcdv3/client_test.go index e9aea31bc8..b46a7eb561 100644 --- a/remoting/etcdv3/client_test.go +++ b/remoting/etcdv3/client_test.go @@ -37,6 +37,8 @@ import ( "google.golang.org/grpc/connectivity" ) +const defaultEtcdV3WorkDir = "/tmp/default-dubbo-go-remote.etcd" + // tests dataset var tests = []struct { input struct { @@ -92,7 +94,7 @@ func (suite *ClientTestSuite) SetupSuite() { cfg := embed.NewConfig() cfg.LPUrls = []url.URL{*lpurl} cfg.LCUrls = []url.URL{*lcurl} - cfg.Dir = "/tmp/default-dubbo-go-remote.etcd" + cfg.Dir = defaultEtcdV3WorkDir e, err := embed.StartEtcd(cfg) if err != nil { t.Fatal(err) @@ -112,7 +114,7 @@ func (suite *ClientTestSuite) SetupSuite() { // stop etcd server func (suite *ClientTestSuite) TearDownSuite() { suite.etcd.Close() - if err := exec.Command("rm", "-rf", "/tmp/default-dubbo-go-remote.etcd").Run(); err != nil { + if err := exec.Command("rm", "-rf", defaultEtcdV3WorkDir).Run(); err != nil { suite.FailNow(err.Error()) } } diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index a81c85abd0..beb83b902e 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -216,8 +216,6 @@ func (c *Client) initStore() error { // try to watch kubernetes pods func (c *Client) maintenanceStatus() error { - c.wg.Add(1) - // try once watcher, err := c.rawClient.CoreV1().Pods(c.ns).Watch(metav1.ListOptions{ LabelSelector: fields.OneTermEqualSelector(DubboIOLabelKey, DubboIOLabelValue).String(), @@ -229,6 +227,7 @@ func (c *Client) maintenanceStatus() error { watcher.Stop() + c.wg.Add(1) // add wg, grace close the client go c.maintenanceStatusLoop() return nil From 5975ca09b8124c278e890fed5703ea5756537b71 Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Sun, 19 Jan 2020 10:21:26 +0800 Subject: [PATCH 13/61] Fix registry concurrent close panic --- registry/kubernetes/registry.go | 10 +++++++--- remoting/etcdv3/listener.go | 2 +- remoting/kubernetes/listener.go | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index 1153078dc4..7880cf3a93 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -71,8 +71,9 @@ type kubernetesRegistry struct { dataListener *dataListener configListener *configurationListener - wg sync.WaitGroup // wg+done for kubernetes client restart - done chan struct{} + wg sync.WaitGroup // wg+done for kubernetes client restart + closeOnce sync.Once // protect the done + done chan struct{} } func (r *kubernetesRegistry) Client() *kubernetes.Client { @@ -156,7 +157,10 @@ func (r *kubernetesRegistry) Destroy() { func (r *kubernetesRegistry) stop() { - close(r.done) + // close will be call concurrent + r.closeOnce.Do(func() { + close(r.done) + }) // close current client r.client.Close() diff --git a/remoting/etcdv3/listener.go b/remoting/etcdv3/listener.go index a4d5805a6d..6c9905a100 100644 --- a/remoting/etcdv3/listener.go +++ b/remoting/etcdv3/listener.go @@ -200,7 +200,7 @@ func (l *EventListener) ListenServiceEvent(key string, listener remoting.DataLis keyList, valueList, err := l.client.getChildren(key) if err != nil { - logger.Errorf("Get new node path {%v} 's content error,message is {%v}", key, perrors.WithMessage(err, "get children")) + logger.Warnf("Get new node path {%v} 's content error,message is {%v}", key, perrors.WithMessage(err, "get children")) } logger.Infof("get key children list %s, keys %v values %v", key, keyList, valueList) diff --git a/remoting/kubernetes/listener.go b/remoting/kubernetes/listener.go index 5c23d7872f..c8cc10f36a 100644 --- a/remoting/kubernetes/listener.go +++ b/remoting/kubernetes/listener.go @@ -171,7 +171,7 @@ func (l *EventListener) ListenServiceEvent(key string, listener remoting.DataLis keyList, valueList, err := l.client.GetChildren(key) if err != nil { - logger.Errorf("Get new node path {%v} 's content error,message is {%v}", key, perrors.WithMessage(err, "get children")) + logger.Warnf("Get new node path {%v} 's content error,message is {%v}", key, perrors.WithMessage(err, "get children")) } logger.Infof("get key children list %s, keys %v values %v", key, keyList, valueList) From 4e5debcafb8451c54032f3a57c8084b3f6c186c7 Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Fri, 13 Mar 2020 14:53:06 +0800 Subject: [PATCH 14/61] Add ut for remote/kubernetes --- remoting/kubernetes/client.go | 65 ++- remoting/kubernetes/client_test.go | 670 +++++++++++++++++++++++++++++ remoting/kubernetes/store.go | 12 +- 3 files changed, 742 insertions(+), 5 deletions(-) create mode 100644 remoting/kubernetes/client_test.go diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index e4e9f8eb8c..794491ebc9 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -22,14 +22,13 @@ import ( "encoding/base64" "encoding/json" "os" - "runtime/debug" "sync" "time" ) import ( perrors "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/types" @@ -106,6 +105,53 @@ func getCurrentNameSpace() (string, error) { return v, nil } +// new mock client +// new a client for test +func newMockClient(namespace string, mockClientGenerator func() (kubernetes.Interface, error)) (*Client, error) { + + rawClient, err := mockClientGenerator() + if err != nil { + return nil, perrors.WithMessage(err, "call mock generator") + } + + currentPodName, err := getCurrentPodName() + if err != nil { + return nil, perrors.WithMessage(err, "get pod name") + } + + ctx, cancel := context.WithCancel(context.Background()) + + c := &Client{ + currentPodName: currentPodName, + ns: namespace, + rawClient: rawClient, + ctx: ctx, + store: newStore(ctx), + cancel: cancel, + } + + currentPod, err := c.initCurrentPod() + if err != nil { + return nil, perrors.WithMessage(err, "init current pod") + } + + // record current status + c.currentPod = currentPod + + // init the store by current pods + if err := c.initStore(); err != nil { + return nil, perrors.WithMessage(err, "init store") + } + + // start kubernetes watch loop + if err := c.maintenanceStatus(); err != nil { + return nil, perrors.WithMessage(err, "maintenance the kubernetes status") + } + + logger.Info("init kubernetes registry success") + return c, nil +} + // newClient // new a client for registry func newClient(namespace string) (*Client, error) { @@ -272,6 +318,7 @@ func (c *Client) maintenanceStatusLoop() { // double check ctx case <-c.ctx.Done(): logger.Info("the kubernetes client stopped") + goto onceWatch // get one element from result-chan case event, ok := <-wc.ResultChan(): @@ -550,12 +597,22 @@ func (c *Client) GetChildren(k string) ([]string, []string, error) { return kList, vList, nil } +// Get +// get k's value from kubernetes-store +func (c *Client) Get(k string) (string, error) { + + objectList, err := c.store.Get(k, false) + if err != nil { + return "", perrors.WithMessagef(err, "get from store on (%s)", k) + } + + return objectList[0].Value, nil +} + // Watch // watch on spec key func (c *Client) Watch(k string) (<-chan *Object, error) { - debug.PrintStack() - w, err := c.store.Watch(k, false) if err != nil { return nil, perrors.WithMessagef(err, "watch on (%s)", k) diff --git a/remoting/kubernetes/client_test.go b/remoting/kubernetes/client_test.go new file mode 100644 index 0000000000..ffd3540402 --- /dev/null +++ b/remoting/kubernetes/client_test.go @@ -0,0 +1,670 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kubernetes + +import ( + "encoding/json" + "os" + "strings" + "sync" + "testing" + "time" +) + +import ( + "github.com/pkg/errors" + "github.com/stretchr/testify/suite" + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" +) + +// tests dataset +var tests = []struct { + input struct { + k string + v string + } +}{ + {input: struct { + k string + v string + }{k: "name", v: "scott.wang"}}, + {input: struct { + k string + v string + }{k: "namePrefix", v: "prefix.scott.wang"}}, + {input: struct { + k string + v string + }{k: "namePrefix1", v: "prefix1.scott.wang"}}, + {input: struct { + k string + v string + }{k: "age", v: "27"}}, +} + +// test dataset prefix +const prefix = "name" + +var clientPodJsonData = `{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "annotations": { + "dubbo.io/annotation": "W3siayI6Ii9kdWJibyIsInYiOiIifSx7ImsiOiIvZHViYm8vY29tLmlrdXJlbnRvLnVzZXIuVXNlclByb3ZpZGVyIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvY29uc3VtZXJzIiwidiI6IiJ9LHsiayI6Ii9kdWJibyIsInYiOiIifSx7ImsiOiIvZHViYm8vY29tLmlrdXJlbnRvLnVzZXIuVXNlclByb3ZpZGVyIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvcHJvdmlkZXJzIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvY29uc3VtZXJzL2NvbnN1bWVyJTNBJTJGJTJGMTcyLjE3LjAuOCUyRlVzZXJQcm92aWRlciUzRmNhdGVnb3J5JTNEY29uc3VtZXJzJTI2ZHViYm8lM0RkdWJib2dvLWNvbnN1bWVyLTIuNi4wJTI2cHJvdG9jb2wlM0RkdWJibyIsInYiOiIifV0=" + }, + "creationTimestamp": "2020-03-13T03:38:57Z", + "labels": { + "dubbo.io/label": "dubbo.io-value" + }, + "name": "client", + "namespace": "default", + "resourceVersion": "2449700", + "selfLink": "/api/v1/namespaces/default/pods/client", + "uid": "3ec394f5-dcc6-49c3-8061-57b4b2b41344" + }, + "spec": { + "containers": [ + { + "env": [ + { + "name": "NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "image": "registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-client", + "imagePullPolicy": "Always", + "name": "client", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "dubbo-sa-token-l2lzh", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "minikube", + "priority": 0, + "restartPolicy": "Never", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "dubbo-sa", + "serviceAccountName": "dubbo-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "dubbo-sa-token-l2lzh", + "secret": { + "defaultMode": 420, + "secretName": "dubbo-sa-token-l2lzh" + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:38:57Z", + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:40:18Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:40:18Z", + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:38:57Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "docker://2870d6abc19ca7fe22ca635ebcfac5d48c6d5550a659bafd74fb48104f6dfe3c", + "image": "registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-client:latest", + "imageID": "docker-pullable://registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-client@sha256:1f075131f708a0d400339e81549d7c4d4ed917ab0b6bd38ef458dd06ad25a559", + "lastState": {}, + "name": "client", + "ready": true, + "restartCount": 0, + "state": { + "running": { + "startedAt": "2020-03-13T03:40:17Z" + } + } + } + ], + "hostIP": "10.0.2.15", + "phase": "Running", + "podIP": "172.17.0.8", + "qosClass": "BestEffort", + "startTime": "2020-03-13T03:38:57Z" + } +} +` + +var server1PodJsonData = `{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "annotations": { + "dubbo.io/annotation": "W3siayI6Ii9kdWJibyIsInYiOiIifSx7ImsiOiIvZHViYm8vY29tLmlrdXJlbnRvLnVzZXIuVXNlclByb3ZpZGVyIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvcHJvdmlkZXJzIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvcHJvdmlkZXJzL2R1YmJvJTNBJTJGJTJGMTcyLjE3LjAuNyUzQTIwMDAwJTJGVXNlclByb3ZpZGVyJTNGYWNjZXNzbG9nJTNEJTI2YW55aG9zdCUzRHRydWUlMjZhcHAudmVyc2lvbiUzRDAuMC4xJTI2YXBwbGljYXRpb24lM0RCRFRTZXJ2aWNlJTI2YmVhbi5uYW1lJTNEVXNlclByb3ZpZGVyJTI2Y2F0ZWdvcnklM0Rwcm92aWRlcnMlMjZjbHVzdGVyJTNEZmFpbG92ZXIlMjZkdWJibyUzRGR1YmJvLXByb3ZpZGVyLWdvbGFuZy0yLjYuMCUyNmVudmlyb25tZW50JTNEZGV2JTI2ZXhlY3V0ZS5saW1pdCUzRCUyNmV4ZWN1dGUubGltaXQucmVqZWN0ZWQuaGFuZGxlciUzRCUyNmdyb3VwJTNEJTI2aW50ZXJmYWNlJTNEY29tLmlrdXJlbnRvLnVzZXIuVXNlclByb3ZpZGVyJTI2aXAlM0QxNzIuMTcuMC43JTI2bG9hZGJhbGFuY2UlM0RyYW5kb20lMjZtZXRob2RzLkdldFVzZXIubG9hZGJhbGFuY2UlM0RyYW5kb20lMjZtZXRob2RzLkdldFVzZXIucmV0cmllcyUzRDElMjZtZXRob2RzLkdldFVzZXIudHBzLmxpbWl0LmludGVydmFsJTNEJTI2bWV0aG9kcy5HZXRVc2VyLnRwcy5saW1pdC5yYXRlJTNEJTI2bWV0aG9kcy5HZXRVc2VyLnRwcy5saW1pdC5zdHJhdGVneSUzRCUyNm1ldGhvZHMuR2V0VXNlci53ZWlnaHQlM0QwJTI2bW9kdWxlJTNEZHViYm9nbyUyQnVzZXItaW5mbyUyQnNlcnZlciUyNm5hbWUlM0RCRFRTZXJ2aWNlJTI2b3JnYW5pemF0aW9uJTNEaWt1cmVudG8uY29tJTI2b3duZXIlM0RaWCUyNnBpZCUzRDEwJTI2cmVnaXN0cnkucm9sZSUzRDMlMjZyZXRyaWVzJTNEJTI2c2VydmljZS5maWx0ZXIlM0RlY2hvJTI1MkN0b2tlbiUyNTJDYWNjZXNzbG9nJTI1MkN0cHMlMjUyQ2V4ZWN1dGUlMjZzaWRlJTNEcHJvdmlkZXIlMjZ0aW1lc3RhbXAlM0QxNTg0MDcwODEwJTI2dHBzLmxpbWl0LmludGVydmFsJTNEJTI2dHBzLmxpbWl0LnJhdGUlM0QlMjZ0cHMubGltaXQucmVqZWN0ZWQuaGFuZGxlciUzRCUyNnRwcy5saW1pdC5zdHJhdGVneSUzRCUyNnRwcy5saW1pdGVyJTNEJTI2dmVyc2lvbiUzRCUyNndhcm11cCUzRDEwMCIsInYiOiIifV0=" + }, + "creationTimestamp": "2020-03-13T03:38:57Z", + "generateName": "server-5b8f9f85c6-", + "labels": { + "dubbo.io/label": "dubbo.io-value", + "pod-template-hash": "5b8f9f85c6", + "role": "server" + }, + "name": "server-5b8f9f85c6-2w5rq", + "namespace": "default", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "server-5b8f9f85c6", + "uid": "65e9d2b0-f286-4b21-ac31-260f1412556b" + } + ], + "resourceVersion": "2449678", + "selfLink": "/api/v1/namespaces/default/pods/server-5b8f9f85c6-2w5rq", + "uid": "ae7497c7-396d-40c5-b53e-5720a696e4ee" + }, + "spec": { + "containers": [ + { + "env": [ + { + "name": "NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "image": "registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-server", + "imagePullPolicy": "Always", + "name": "server", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "dubbo-sa-token-l2lzh", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "minikube", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "dubbo-sa", + "serviceAccountName": "dubbo-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "dubbo-sa-token-l2lzh", + "secret": { + "defaultMode": 420, + "secretName": "dubbo-sa-token-l2lzh" + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:38:57Z", + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:40:10Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:40:10Z", + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:38:57Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "docker://88144bc6eabf783c0954c2e078a7270c8a0246d6f8af081dfcc0956e8c3cd2de", + "image": "registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-server:latest", + "imageID": "docker-pullable://registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-server@sha256:60654ddba3a16ca3de52c8e30650a4c1d8b2ed8f8542af489b7a5a459e46fe6b", + "lastState": {}, + "name": "server", + "ready": true, + "restartCount": 0, + "state": { + "running": { + "startedAt": "2020-03-13T03:40:10Z" + } + } + } + ], + "hostIP": "10.0.2.15", + "phase": "Running", + "podIP": "172.17.0.7", + "qosClass": "BestEffort", + "startTime": "2020-03-13T03:38:57Z" + } +} +` + +var server2PodJsonData = `{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "annotations": { + "dubbo.io/annotation": "W3siayI6Ii9kdWJibyIsInYiOiIifSx7ImsiOiIvZHViYm8vY29tLmlrdXJlbnRvLnVzZXIuVXNlclByb3ZpZGVyIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvcHJvdmlkZXJzIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvcHJvdmlkZXJzL2R1YmJvJTNBJTJGJTJGMTcyLjE3LjAuNiUzQTIwMDAwJTJGVXNlclByb3ZpZGVyJTNGYWNjZXNzbG9nJTNEJTI2YW55aG9zdCUzRHRydWUlMjZhcHAudmVyc2lvbiUzRDAuMC4xJTI2YXBwbGljYXRpb24lM0RCRFRTZXJ2aWNlJTI2YmVhbi5uYW1lJTNEVXNlclByb3ZpZGVyJTI2Y2F0ZWdvcnklM0Rwcm92aWRlcnMlMjZjbHVzdGVyJTNEZmFpbG92ZXIlMjZkdWJibyUzRGR1YmJvLXByb3ZpZGVyLWdvbGFuZy0yLjYuMCUyNmVudmlyb25tZW50JTNEZGV2JTI2ZXhlY3V0ZS5saW1pdCUzRCUyNmV4ZWN1dGUubGltaXQucmVqZWN0ZWQuaGFuZGxlciUzRCUyNmdyb3VwJTNEJTI2aW50ZXJmYWNlJTNEY29tLmlrdXJlbnRvLnVzZXIuVXNlclByb3ZpZGVyJTI2aXAlM0QxNzIuMTcuMC42JTI2bG9hZGJhbGFuY2UlM0RyYW5kb20lMjZtZXRob2RzLkdldFVzZXIubG9hZGJhbGFuY2UlM0RyYW5kb20lMjZtZXRob2RzLkdldFVzZXIucmV0cmllcyUzRDElMjZtZXRob2RzLkdldFVzZXIudHBzLmxpbWl0LmludGVydmFsJTNEJTI2bWV0aG9kcy5HZXRVc2VyLnRwcy5saW1pdC5yYXRlJTNEJTI2bWV0aG9kcy5HZXRVc2VyLnRwcy5saW1pdC5zdHJhdGVneSUzRCUyNm1ldGhvZHMuR2V0VXNlci53ZWlnaHQlM0QwJTI2bW9kdWxlJTNEZHViYm9nbyUyQnVzZXItaW5mbyUyQnNlcnZlciUyNm5hbWUlM0RCRFRTZXJ2aWNlJTI2b3JnYW5pemF0aW9uJTNEaWt1cmVudG8uY29tJTI2b3duZXIlM0RaWCUyNnBpZCUzRDEwJTI2cmVnaXN0cnkucm9sZSUzRDMlMjZyZXRyaWVzJTNEJTI2c2VydmljZS5maWx0ZXIlM0RlY2hvJTI1MkN0b2tlbiUyNTJDYWNjZXNzbG9nJTI1MkN0cHMlMjUyQ2V4ZWN1dGUlMjZzaWRlJTNEcHJvdmlkZXIlMjZ0aW1lc3RhbXAlM0QxNTg0MDcwODA5JTI2dHBzLmxpbWl0LmludGVydmFsJTNEJTI2dHBzLmxpbWl0LnJhdGUlM0QlMjZ0cHMubGltaXQucmVqZWN0ZWQuaGFuZGxlciUzRCUyNnRwcy5saW1pdC5zdHJhdGVneSUzRCUyNnRwcy5saW1pdGVyJTNEJTI2dmVyc2lvbiUzRCUyNndhcm11cCUzRDEwMCIsInYiOiIifV0=" + }, + "creationTimestamp": "2020-03-13T03:38:57Z", + "generateName": "server-5b8f9f85c6-", + "labels": { + "dubbo.io/label": "dubbo.io-value", + "pod-template-hash": "5b8f9f85c6", + "role": "server" + }, + "name": "server-5b8f9f85c6-xk5md", + "namespace": "default", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "ReplicaSet", + "name": "server-5b8f9f85c6", + "uid": "65e9d2b0-f286-4b21-ac31-260f1412556b" + } + ], + "resourceVersion": "2449667", + "selfLink": "/api/v1/namespaces/default/pods/server-5b8f9f85c6-xk5md", + "uid": "9e59e164-6620-473b-a983-472ebc1120e9" + }, + "spec": { + "containers": [ + { + "env": [ + { + "name": "NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "image": "registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-server", + "imagePullPolicy": "Always", + "name": "server", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "dubbo-sa-token-l2lzh", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "minikube", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "dubbo-sa", + "serviceAccountName": "dubbo-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "dubbo-sa-token-l2lzh", + "secret": { + "defaultMode": 420, + "secretName": "dubbo-sa-token-l2lzh" + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:38:57Z", + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:40:09Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:40:09Z", + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:38:57Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "docker://442dc055392cc720b6a6eb0bc4105f13ea86e63cbdced5c83f15463bc61add76", + "image": "registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-server:latest", + "imageID": "docker-pullable://registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-server@sha256:60654ddba3a16ca3de52c8e30650a4c1d8b2ed8f8542af489b7a5a459e46fe6b", + "lastState": {}, + "name": "server", + "ready": true, + "restartCount": 0, + "state": { + "running": { + "startedAt": "2020-03-13T03:40:09Z" + } + } + } + ], + "hostIP": "10.0.2.15", + "phase": "Running", + "podIP": "172.17.0.6", + "qosClass": "BestEffort", + "startTime": "2020-03-13T03:38:57Z" + } +}` + +type KubernetesClientTestSuite struct { + suite.Suite + + client *Client + + currentPod v1.Pod + fakeServerPod1 v1.Pod + fakeServerPod2 v1.Pod +} + +func (s *KubernetesClientTestSuite) SetupSuite() { + + t := s.T() + + // 1. install test data + if err := json.Unmarshal([]byte(clientPodJsonData), &s.currentPod); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(server1PodJsonData), &s.fakeServerPod1); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(server2PodJsonData), &s.fakeServerPod2); err != nil { + t.Fatal(err) + } + + // 2. set downward-api inject env + if err := os.Setenv(podNameKey, s.currentPod.GetName()); err != nil { + t.Fatal(err) + } + if err := os.Setenv(nameSpaceKey, s.currentPod.GetNamespace()); err != nil { + t.Fatal(err) + } +} + +func (s *KubernetesClientTestSuite) TearDownSuite() { + s.client.Close() + os.Unsetenv(podNameKey) + os.Unsetenv(nameSpaceKey) +} + +func (s *KubernetesClientTestSuite) SetupTest() { + + t := s.T() + var err error + s.client, err = newMockClient(s.currentPod.GetNamespace(), func() (kubernetes.Interface, error) { + + out := fake.NewSimpleClientset() + + // mock current pod + if _, err := out.CoreV1().Pods(s.currentPod.GetNamespace()).Create(&s.currentPod); err != nil { + return nil, errors.WithMessage(err, "mock current pod ") + } + return out, nil + }) + if err != nil { + t.Fatal(err) + } +} + +func (s *KubernetesClientTestSuite) TestClientValid() { + + t := s.T() + + if s.client.Valid() != true { + t.Fatal("client is not valid") + } + s.client.Close() + + if s.client.Valid() != false { + t.Fatal("client is valid") + } +} + +func (s *KubernetesClientTestSuite) TestClientDone() { + + t := s.T() + + go func() { + time.Sleep(time.Second) + s.client.Close() + }() + + <-s.client.Done() + + if s.client.Valid() == true { + t.Fatal("client should be invalid then") + } +} + +func (s *KubernetesClientTestSuite) TestClientCreateKV() { + + t := s.T() + defer s.client.Close() + + for _, tc := range tests { + + k := tc.input.k + v := tc.input.v + + if err := s.client.Create(k, v); err != nil { + t.Fatal(err) + } + + } +} + +func (s *KubernetesClientTestSuite) TestClientGetChildrenKVList() { + + t := s.T() + defer s.client.Close() + + expect := make(map[string]string) + got := make(map[string]string) + + for _, tc := range tests { + + k := tc.input.k + v := tc.input.v + + if strings.Contains(k, prefix) { + expect[k] = v + } + + if err := s.client.Create(k, v); err != nil { + t.Fatal(err) + } + } + + kList, vList, err := s.client.GetChildren(prefix) + if err != nil { + t.Fatal(err) + } + + for i := 0; i < len(kList); i++ { + got[kList[i]] = vList[i] + } + + for expectK, expectV := range expect { + + if got[expectK] != expectV { + t.Fatalf("expect {%s: %s} but got {%s: %v}", expectK, expectV, expectK, got[expectK]) + } + } + +} + +func (s *KubernetesClientTestSuite) TestClientWatch() { + + t := s.T() + + wg := sync.WaitGroup{} + wg.Add(1) + + go func() { + + defer wg.Done() + + wc, err := s.client.WatchWithPrefix(prefix) + if err != nil { + t.Fatal(err) + } + + for e := range wc { + t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) + } + + }() + + for _, tc := range tests { + + k := tc.input.k + v := tc.input.v + + if err := s.client.Create(k, v); err != nil { + t.Fatal(err) + } + } + + s.client.Close() + wg.Wait() +} + +func TestKubernetesClient(t *testing.T) { + suite.Run(t, new(KubernetesClientTestSuite)) +} diff --git a/remoting/kubernetes/store.go b/remoting/kubernetes/store.go index d7f35ced63..5feab9f0b0 100644 --- a/remoting/kubernetes/store.go +++ b/remoting/kubernetes/store.go @@ -76,7 +76,7 @@ type Store interface { // put the object to the store Put(object *Object) error // if prefix is false, - // the len([]*Object) == 0 + // the len([]*Object) == 1 Get(key string, prefix bool) ([]*Object, error) // watch the spec key or key prefix Watch(key string, prefix bool) (Watcher, error) @@ -166,6 +166,16 @@ func (s *storeImpl) Put(object *Object) error { if object.EventType == Delete { delete(s.cache, object.Key) } else { + + old, ok := s.cache[object.Key] + if ok { + if old.Value == object.Value { + // already have this k/v pair + return nil + } + } + + // refresh the object s.cache[object.Key] = object } From 60eaf55c36ebff53e826f42c3dee45a9bfa9ab6b Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Fri, 13 Mar 2020 14:56:20 +0800 Subject: [PATCH 15/61] Delete unused method --- registry/kubernetes/listener_test.go | 3 +-- registry/kubernetes/registry.go | 1 + remoting/kubernetes/client.go | 12 ------------ remoting/kubernetes/store_test.go | 2 +- 4 files changed, 3 insertions(+), 15 deletions(-) diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go index 866f594d31..152f2adf92 100644 --- a/registry/kubernetes/listener_test.go +++ b/registry/kubernetes/listener_test.go @@ -18,7 +18,6 @@ package kubernetes import ( - "context" "testing" ) @@ -34,7 +33,7 @@ import ( func Test_DataChange(t *testing.T) { listener := NewRegistryDataListener(&MockDataListener{}) - url, _ := common.NewURL(context.TODO(), "jsonrpc%3A%2F%2F127.0.0.1%3A20001%2Fcom.ikurento.user.UserProvider%3Fanyhost%3Dtrue%26app.version%3D0.0.1%26application%3DBDTService%26category%3Dproviders%26cluster%3Dfailover%26dubbo%3Ddubbo-provider-golang-2.6.0%26environment%3Ddev%26group%3D%26interface%3Dcom.ikurento.user.UserProvider%26ip%3D10.32.20.124%26loadbalance%3Drandom%26methods.GetUser.loadbalance%3Drandom%26methods.GetUser.retries%3D1%26methods.GetUser.weight%3D0%26module%3Ddubbogo%2Buser-info%2Bserver%26name%3DBDTService%26organization%3Dikurento.com%26owner%3DZX%26pid%3D74500%26retries%3D0%26service.filter%3Decho%26side%3Dprovider%26timestamp%3D1560155407%26version%3D%26warmup%3D100") + url, _ := common.NewURL("jsonrpc%3A%2F%2F127.0.0.1%3A20001%2Fcom.ikurento.user.UserProvider%3Fanyhost%3Dtrue%26app.version%3D0.0.1%26application%3DBDTService%26category%3Dproviders%26cluster%3Dfailover%26dubbo%3Ddubbo-provider-golang-2.6.0%26environment%3Ddev%26group%3D%26interface%3Dcom.ikurento.user.UserProvider%26ip%3D10.32.20.124%26loadbalance%3Drandom%26methods.GetUser.loadbalance%3Drandom%26methods.GetUser.retries%3D1%26methods.GetUser.weight%3D0%26module%3Ddubbogo%2Buser-info%2Bserver%26name%3DBDTService%26organization%3Dikurento.com%26owner%3DZX%26pid%3D74500%26retries%3D0%26service.filter%3Decho%26side%3Dprovider%26timestamp%3D1560155407%26version%3D%26warmup%3D100") listener.AddInterestedURL(&url) int := listener.DataChange(remoting.Event{Path: "/dubbo/com.ikurento.user.UserProvider/providers/jsonrpc%3A%2F%2F127.0.0.1%3A20001%2Fcom.ikurento.user.UserProvider%3Fanyhost%3Dtrue%26app.version%3D0.0.1%26application%3DBDTService%26category%3Dproviders%26cluster%3Dfailover%26dubbo%3Ddubbo-provider-golang-2.6.0%26environment%3Ddev%26group%3D%26interface%3Dcom.ikurento.user.UserProvider%26ip%3D10.32.20.124%26loadbalance%3Drandom%26methods.GetUser.loadbalance%3Drandom%26methods.GetUser.retries%3D1%26methods.GetUser.weight%3D0%26module%3Ddubbogo%2Buser-info%2Bserver%26name%3DBDTService%26organization%3Dikurento.com%26owner%3DZX%26pid%3D74500%26retries%3D0%26service.filter%3Decho%26side%3Dprovider%26timestamp%3D1560155407%26version%3D%26warmup%3D100"}) assert.Equal(t, true, int) diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index 7880cf3a93..61ac32fb42 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -61,6 +61,7 @@ func init() { type kubernetesRegistry struct { *common.URL birth int64 // time of file birth, seconds since Epoch; 0 if unknown + registry.BaseRegistry cltLock sync.Mutex client *kubernetes.Client diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index 794491ebc9..dd1c0a3911 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -597,18 +597,6 @@ func (c *Client) GetChildren(k string) ([]string, []string, error) { return kList, vList, nil } -// Get -// get k's value from kubernetes-store -func (c *Client) Get(k string) (string, error) { - - objectList, err := c.store.Get(k, false) - if err != nil { - return "", perrors.WithMessagef(err, "get from store on (%s)", k) - } - - return objectList[0].Value, nil -} - // Watch // watch on spec key func (c *Client) Watch(k string) (<-chan *Object, error) { diff --git a/remoting/kubernetes/store_test.go b/remoting/kubernetes/store_test.go index e4acee4b38..2bc8b277a7 100644 --- a/remoting/kubernetes/store_test.go +++ b/remoting/kubernetes/store_test.go @@ -28,7 +28,7 @@ import ( func TestStore(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() s := newStore(ctx) From 34ebc722ac93885720d088053247e4e841ab83c7 Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Fri, 13 Mar 2020 15:12:30 +0800 Subject: [PATCH 16/61] adapte for new registry --- registry/kubernetes/listener.go | 3 +- registry/kubernetes/registry.go | 276 ++++---------------------------- remoting/kubernetes/facade.go | 8 +- 3 files changed, 37 insertions(+), 250 deletions(-) diff --git a/registry/kubernetes/listener.go b/registry/kubernetes/listener.go index 0a494a422d..7c59a36308 100644 --- a/registry/kubernetes/listener.go +++ b/registry/kubernetes/listener.go @@ -18,7 +18,6 @@ package kubernetes import ( - "context" "strings" ) @@ -55,7 +54,7 @@ func (l *dataListener) DataChange(eventType remoting.Event) bool { return false } url := eventType.Path[index+len("/providers/"):] - serviceURL, err := common.NewURL(context.Background(), url) + serviceURL, err := common.NewURL(url) if err != nil { logger.Warnf("Listen NewURL(r{%s}) = error{%v}", eventType.Path, err) return false diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index 61ac32fb42..c68f0963a9 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -19,17 +19,14 @@ package kubernetes import ( "fmt" - "net/url" "os" "path" - "strconv" "strings" "sync" - "time" ) import ( - gxnet "github.com/dubbogo/gost/net" + "github.com/dubbogo/gost/net" perrors "github.com/pkg/errors" ) @@ -48,8 +45,7 @@ var ( ) const ( - Name = "kubernetes" - RegistryConnDelay = 3 + Name = "kubernetes" ) func init() { @@ -59,22 +55,13 @@ func init() { } type kubernetesRegistry struct { - *common.URL - birth int64 // time of file birth, seconds since Epoch; 0 if unknown registry.BaseRegistry - - cltLock sync.Mutex - client *kubernetes.Client - services map[string]common.URL // service name + protocol -> service config - + cltLock sync.Mutex + client *kubernetes.Client listenerLock sync.Mutex listener *kubernetes.EventListener dataListener *dataListener configListener *configurationListener - - wg sync.WaitGroup // wg+done for kubernetes client restart - closeOnce sync.Once // protect the done - done chan struct{} } func (r *kubernetesRegistry) Client() *kubernetes.Client { @@ -86,129 +73,19 @@ func (r *kubernetesRegistry) SetClient(client *kubernetes.Client) { func (r *kubernetesRegistry) ClientLock() *sync.Mutex { return &r.cltLock } -func (r *kubernetesRegistry) WaitGroup() *sync.WaitGroup { - return &r.wg -} -func (r *kubernetesRegistry) GetDone() chan struct{} { - return r.done -} -func (r *kubernetesRegistry) RestartCallBack() bool { - - services := []common.URL{} - for _, confIf := range r.services { - services = append(services, confIf) - } - - for _, confIf := range services { - err := r.Register(confIf) - if err != nil { - logger.Errorf("(kubernetesProviderRegistry)register(conf{%#v}) = error{%#v}", - confIf, perrors.WithStack(err)) - return false - } - logger.Infof("success to re-register service :%v", confIf.Key()) - } - return true -} - -func newKubernetesRegistry(url *common.URL) (registry.Registry, error) { - r := &kubernetesRegistry{ - URL: url, - birth: time.Now().UnixNano(), - done: make(chan struct{}), - services: make(map[string]common.URL), - } - - if err := kubernetes.ValidateClient(r); err != nil { - return nil, err - } - - r.wg.Add(1) - go kubernetes.HandleClientRestart(r) - - r.listener = kubernetes.NewEventListener(r.client) - r.configListener = NewConfigurationListener(r) - r.dataListener = NewRegistryDataListener(r.configListener) - - return r, nil -} - -func (r *kubernetesRegistry) GetUrl() common.URL { - return *r.URL -} - -func (r *kubernetesRegistry) IsAvailable() bool { - - select { - case <-r.done: - return false - default: - return true - } -} - -func (r *kubernetesRegistry) Destroy() { - - if r.configListener != nil { - r.configListener.Close() - } - r.stop() -} - -func (r *kubernetesRegistry) stop() { - - // close will be call concurrent - r.closeOnce.Do(func() { - close(r.done) - }) - - // close current client +func (r *kubernetesRegistry) CloseAndNilClient() { r.client.Close() - - r.cltLock.Lock() r.client = nil - r.services = nil - r.cltLock.Unlock() } -func (r *kubernetesRegistry) Register(svc common.URL) error { - - role, err := strconv.Atoi(r.URL.GetParam(constant.ROLE_KEY, "")) - if err != nil { - return perrors.WithMessage(err, "get registry role") - } - - r.cltLock.Lock() - if _, ok := r.services[svc.Key()]; ok { - r.cltLock.Unlock() - return perrors.New(fmt.Sprintf("Path{%s} has been registered", svc.Path)) - } - r.cltLock.Unlock() - - switch role { - case common.PROVIDER: - logger.Debugf("(provider register )Register(conf{%#v})", svc) - if err := r.registerProvider(svc); err != nil { - return perrors.WithMessage(err, "register provider") - } - case common.CONSUMER: - logger.Debugf("(consumer register )Register(conf{%#v})", svc) - if err := r.registerConsumer(svc); err != nil { - return perrors.WithMessage(err, "register consumer") - } - default: - return perrors.New(fmt.Sprintf("unknown role %d", role)) +func (r *kubernetesRegistry) CloseListener() { + if r.configListener != nil { + r.configListener.Close() } - - r.cltLock.Lock() - r.services[svc.Key()] = svc - r.cltLock.Unlock() - return nil } -func (r *kubernetesRegistry) createDirIfNotExist(k string) error { - +func (r *kubernetesRegistry) CreatePath(k string) error { var tmpPath string for _, str := range strings.Split(k, "/")[1:] { tmpPath = path.Join(tmpPath, "/", str) @@ -220,87 +97,11 @@ func (r *kubernetesRegistry) createDirIfNotExist(k string) error { return nil } -func (r *kubernetesRegistry) registerConsumer(svc common.URL) error { - - consumersNode := fmt.Sprintf("/dubbo/%s/%s", svc.Service(), common.DubboNodes[common.CONSUMER]) - if err := r.createDirIfNotExist(consumersNode); err != nil { - logger.Errorf("kubernetes client create path %s: %v", consumersNode, err) - return perrors.WithMessage(err, "kubernetes create consumer nodes") - } - - // NOTICE kubernetes && etcdv3 not need create provider metadata dir in consumer logic - //providersNode := fmt.Sprintf("/dubbo/%s/%s", svc.Service(), common.DubboNodes[common.PROVIDER]) - //if err := r.createDirIfNotExist(providersNode); err != nil { - // return perrors.WithMessage(err, "create provider node") - //} - - params := url.Values{} - - params.Add("protocol", svc.Protocol) - - params.Add("category", (common.RoleType(common.CONSUMER)).String()) - params.Add("dubbo", "dubbogo-consumer-"+constant.Version) - - encodedURL := url.QueryEscape(fmt.Sprintf("consumer://%s%s?%s", localIP, svc.Path, params.Encode())) - dubboPath := fmt.Sprintf("/dubbo/%s/%s", svc.Service(), (common.RoleType(common.CONSUMER)).String()) - if err := r.client.Create(path.Join(dubboPath, encodedURL), ""); err != nil { - return perrors.WithMessagef(err, "create k/v in kubernetes (path:%s, url:%s)", dubboPath, encodedURL) - } - - return nil -} - -func (r *kubernetesRegistry) registerProvider(svc common.URL) error { - - if len(svc.Path) == 0 || len(svc.Methods) == 0 { - return perrors.New(fmt.Sprintf("service path %s or service method %s", svc.Path, svc.Methods)) - } - - var ( - urlPath string - encodedURL string - dubboPath string - ) - - providersNode := fmt.Sprintf("/dubbo/%s/%s", svc.Service(), common.DubboNodes[common.PROVIDER]) - if err := r.createDirIfNotExist(providersNode); err != nil { - return perrors.WithMessage(err, "create provider node") - } - - params := url.Values{} - - svc.RangeParams(func(key, value string) bool { - params[key] = []string{value} - return true - }) - params.Add("pid", processID) - params.Add("ip", localIP) - params.Add("anyhost", "true") - params.Add("category", (common.RoleType(common.PROVIDER)).String()) - params.Add("dubbo", "dubbo-provider-golang-"+constant.Version) - params.Add("side", (common.RoleType(common.PROVIDER)).Role()) - - logger.Debugf("provider url params:%#v", params) - var host string - if len(svc.Ip) == 0 { - host = localIP + ":" + svc.Port - } else { - host = svc.Ip + ":" + svc.Port - } - - urlPath = svc.Path - - encodedURL = url.QueryEscape(fmt.Sprintf("%s://%s%s?%s", svc.Protocol, host, urlPath, params.Encode())) - dubboPath = fmt.Sprintf("/dubbo/%s/%s", svc.Service(), (common.RoleType(common.PROVIDER)).String()) - - if err := r.client.Create(path.Join(dubboPath, encodedURL), ""); err != nil { - return perrors.WithMessagef(err, "create k/v in kubernetes (path:%s, url:%s)", dubboPath, encodedURL) - } - - return nil +func (r *kubernetesRegistry) DoRegister(root string, node string) error { + return r.client.Create(path.Join(root, node), "") } -func (r *kubernetesRegistry) subscribe(svc *common.URL) (registry.Listener, error) { +func (r *kubernetesRegistry) DoSubscribe(svc *common.URL) (registry.Listener, error) { var ( configListener *configurationListener @@ -321,12 +122,7 @@ func (r *kubernetesRegistry) subscribe(svc *common.URL) (registry.Listener, erro listener := kubernetes.NewEventListener(r.client) r.listenerLock.Lock() - // NOTICE: - // double-check the listener - // if r.listener already be assigned, discard the new value - if r.listener == nil { - r.listener = listener - } + r.listener = listener r.listenerLock.Unlock() } @@ -339,36 +135,28 @@ func (r *kubernetesRegistry) subscribe(svc *common.URL) (registry.Listener, erro return configListener, nil } -//subscribe from registry -func (r *kubernetesRegistry) Subscribe(url *common.URL, notifyListener registry.NotifyListener) { - for { - if !r.IsAvailable() { - logger.Warnf("event listener game over.") - return - } +func (r *kubernetesRegistry) InitListeners() { + r.listener = kubernetes.NewEventListener(r.client) + r.configListener = NewConfigurationListener(r) + r.dataListener = NewRegistryDataListener(r.configListener) +} - listener, err := r.subscribe(url) - if err != nil { - if !r.IsAvailable() { - logger.Warnf("event listener game over.") - return - } - logger.Warnf("getListener() = err:%v", perrors.WithStack(err)) - time.Sleep(time.Duration(RegistryConnDelay) * time.Second) - continue - } +func newKubernetesRegistry(url *common.URL) (registry.Registry, error) { - for { - if serviceEvent, err := listener.Next(); err != nil { - logger.Warnf("Selector.watch() = error{%v}", perrors.WithStack(err)) - listener.Close() - return - } else { - logger.Infof("update begin, service event: %v", serviceEvent.String()) - notifyListener.Notify(serviceEvent) - } + // actually, kubernetes use in-cluster config, + r := &kubernetesRegistry{} - } + r.InitBaseRegistry(url, r) + if err := kubernetes.ValidateClient(r); err != nil { + return nil, err } + + r.WaitGroup().Add(1) + go kubernetes.HandleClientRestart(r) + r.InitListeners() + + logger.Debugf("the kubernetes registry started") + + return r, nil } diff --git a/remoting/kubernetes/facade.go b/remoting/kubernetes/facade.go index 0ab689d517..17a4f41767 100644 --- a/remoting/kubernetes/facade.go +++ b/remoting/kubernetes/facade.go @@ -40,8 +40,8 @@ type clientFacade interface { Client() *Client SetClient(*Client) ClientLock() *sync.Mutex - WaitGroup() *sync.WaitGroup - GetDone() chan struct{} + WaitGroup() *sync.WaitGroup //for wait group control, etcd client listener & etcd client container + Done() chan struct{} //for etcd client control RestartCallBack() bool common.Node } @@ -57,7 +57,7 @@ func HandleClientRestart(r clientFacade) { LOOP: for { select { - case <-r.GetDone(): + case <-r.Done(): logger.Warnf("(KubernetesProviderRegistry)reconnectKubernetes goroutine exit now...") break LOOP // re-register all services @@ -71,7 +71,7 @@ LOOP: failTimes = 0 for { select { - case <-r.GetDone(): + case <-r.Done(): logger.Warnf("(KubernetesProviderRegistry)reconnectKubernetes Registry goroutine exit now...") break LOOP case <-getty.GetTimeWheel().After(timeSecondDuration(failTimes * ConnDelay)): // avoid connect frequent From 29c88880e226028bcf3387cd954e0da6d4b64c31 Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Fri, 13 Mar 2020 15:59:17 +0800 Subject: [PATCH 17/61] Add ut for registry/kubernetes --- registry/etcdv3/registry_test.go | 4 +- registry/kubernetes/listener.go | 18 +- registry/kubernetes/listener_test.go | 220 ++++++++++++++++++- registry/kubernetes/registry.go | 21 ++ registry/kubernetes/registry_test.go | 77 ++++++- remoting/kubernetes/client.go | 10 +- remoting/kubernetes/client_test.go | 307 +-------------------------- 7 files changed, 332 insertions(+), 325 deletions(-) diff --git a/registry/etcdv3/registry_test.go b/registry/etcdv3/registry_test.go index 6e26a8f3fc..dc4e382979 100644 --- a/registry/etcdv3/registry_test.go +++ b/registry/etcdv3/registry_test.go @@ -98,7 +98,7 @@ func (suite *RegistryTestSuite) TestSubscribe() { assert.Regexp(t, ".*ServiceEvent{Action{add}.*", serviceEvent.String()) } -func (suite *RegistryTestSuite) TestConsumerDestory() { +func (suite *RegistryTestSuite) TestConsumerDestroy() { t := suite.T() url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) @@ -117,7 +117,7 @@ func (suite *RegistryTestSuite) TestConsumerDestory() { } -func (suite *RegistryTestSuite) TestProviderDestory() { +func (suite *RegistryTestSuite) TestProviderDestroy() { t := suite.T() reg := initRegistry(t) diff --git a/registry/kubernetes/listener.go b/registry/kubernetes/listener.go index 7c59a36308..1f4f2ed215 100644 --- a/registry/kubernetes/listener.go +++ b/registry/kubernetes/listener.go @@ -38,6 +38,7 @@ type dataListener struct { listener config_center.ConfigurationListener } +// NewRegistryDataListener ... func NewRegistryDataListener(listener config_center.ConfigurationListener) *dataListener { return &dataListener{listener: listener, interestedURL: []*common.URL{}} } @@ -48,12 +49,7 @@ func (l *dataListener) AddInterestedURL(url *common.URL) { func (l *dataListener) DataChange(eventType remoting.Event) bool { - index := strings.Index(eventType.Path, "/providers/") - if index == -1 { - logger.Warn("Listen with no url, event.path={%v}", eventType.Path) - return false - } - url := eventType.Path[index+len("/providers/"):] + url := eventType.Path[strings.Index(eventType.Path, "/providers/")+len("/providers/"):] serviceURL, err := common.NewURL(url) if err != nil { logger.Warnf("Listen NewURL(r{%s}) = error{%v}", eventType.Path, err) @@ -81,11 +77,13 @@ type configurationListener struct { events chan *config_center.ConfigChangeEvent } +// NewConfigurationListener for listening the event of kubernetes. func NewConfigurationListener(reg *kubernetesRegistry) *configurationListener { // add a new waiter - reg.wg.Add(1) + reg.WaitGroup().Add(1) return &configurationListener{registry: reg, events: make(chan *config_center.ConfigChangeEvent, 32)} } + func (l *configurationListener) Process(configType *config_center.ConfigChangeEvent) { l.events <- configType } @@ -93,7 +91,7 @@ func (l *configurationListener) Process(configType *config_center.ConfigChangeEv func (l *configurationListener) Next() (*registry.ServiceEvent, error) { for { select { - case <-l.registry.done: + case <-l.registry.Done(): logger.Warnf("listener's kubernetes client connection is broken, so kubernetes event listener exit now.") return nil, perrors.New("listener stopped") @@ -101,7 +99,7 @@ func (l *configurationListener) Next() (*registry.ServiceEvent, error) { logger.Infof("got kubernetes event %#v", e) if e.ConfigType == remoting.EventTypeDel { select { - case <-l.registry.done: + case <-l.registry.Done(): logger.Warnf("update @result{%s}. But its connection to registry is invalid", e.Value) default: } @@ -112,5 +110,5 @@ func (l *configurationListener) Next() (*registry.ServiceEvent, error) { } } func (l *configurationListener) Close() { - l.registry.wg.Done() + l.registry.WaitGroup().Done() } diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go index 152f2adf92..16bbbf8c7a 100644 --- a/registry/kubernetes/listener_test.go +++ b/registry/kubernetes/listener_test.go @@ -18,19 +18,159 @@ package kubernetes import ( + "encoding/json" + "os" + "strconv" "testing" ) import ( "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" ) import ( "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/common/constant" "github.com/apache/dubbo-go/config_center" "github.com/apache/dubbo-go/remoting" ) +var clientPodJsonData = `{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "annotations": { + "dubbo.io/annotation": "W3siayI6Ii9kdWJibyIsInYiOiIifSx7ImsiOiIvZHViYm8vY29tLmlrdXJlbnRvLnVzZXIuVXNlclByb3ZpZGVyIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvY29uc3VtZXJzIiwidiI6IiJ9LHsiayI6Ii9kdWJibyIsInYiOiIifSx7ImsiOiIvZHViYm8vY29tLmlrdXJlbnRvLnVzZXIuVXNlclByb3ZpZGVyIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvcHJvdmlkZXJzIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvY29uc3VtZXJzL2NvbnN1bWVyJTNBJTJGJTJGMTcyLjE3LjAuOCUyRlVzZXJQcm92aWRlciUzRmNhdGVnb3J5JTNEY29uc3VtZXJzJTI2ZHViYm8lM0RkdWJib2dvLWNvbnN1bWVyLTIuNi4wJTI2cHJvdG9jb2wlM0RkdWJibyIsInYiOiIifV0=" + }, + "creationTimestamp": "2020-03-13T03:38:57Z", + "labels": { + "dubbo.io/label": "dubbo.io-value" + }, + "name": "client", + "namespace": "default", + "resourceVersion": "2449700", + "selfLink": "/api/v1/namespaces/default/pods/client", + "uid": "3ec394f5-dcc6-49c3-8061-57b4b2b41344" + }, + "spec": { + "containers": [ + { + "env": [ + { + "name": "NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "image": "registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-client", + "imagePullPolicy": "Always", + "name": "client", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "dubbo-sa-token-l2lzh", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "nodeName": "minikube", + "priority": 0, + "restartPolicy": "Never", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "dubbo-sa", + "serviceAccountName": "dubbo-sa", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "dubbo-sa-token-l2lzh", + "secret": { + "defaultMode": 420, + "secretName": "dubbo-sa-token-l2lzh" + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:38:57Z", + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:40:18Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:40:18Z", + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2020-03-13T03:38:57Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "docker://2870d6abc19ca7fe22ca635ebcfac5d48c6d5550a659bafd74fb48104f6dfe3c", + "image": "registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-client:latest", + "imageID": "docker-pullable://registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-client@sha256:1f075131f708a0d400339e81549d7c4d4ed917ab0b6bd38ef458dd06ad25a559", + "lastState": {}, + "name": "client", + "ready": true, + "restartCount": 0, + "state": { + "running": { + "startedAt": "2020-03-13T03:40:17Z" + } + } + } + ], + "hostIP": "10.0.2.15", + "phase": "Running", + "podIP": "172.17.0.8", + "qosClass": "BestEffort", + "startTime": "2020-03-13T03:38:57Z" + } +} +` + func Test_DataChange(t *testing.T) { listener := NewRegistryDataListener(&MockDataListener{}) url, _ := common.NewURL("jsonrpc%3A%2F%2F127.0.0.1%3A20001%2Fcom.ikurento.user.UserProvider%3Fanyhost%3Dtrue%26app.version%3D0.0.1%26application%3DBDTService%26category%3Dproviders%26cluster%3Dfailover%26dubbo%3Ddubbo-provider-golang-2.6.0%26environment%3Ddev%26group%3D%26interface%3Dcom.ikurento.user.UserProvider%26ip%3D10.32.20.124%26loadbalance%3Drandom%26methods.GetUser.loadbalance%3Drandom%26methods.GetUser.retries%3D1%26methods.GetUser.weight%3D0%26module%3Ddubbogo%2Buser-info%2Bserver%26name%3DBDTService%26organization%3Dikurento.com%26owner%3DZX%26pid%3D74500%26retries%3D0%26service.filter%3Decho%26side%3Dprovider%26timestamp%3D1560155407%26version%3D%26warmup%3D100") @@ -39,8 +179,84 @@ func Test_DataChange(t *testing.T) { assert.Equal(t, true, int) } -type MockDataListener struct { +type MockDataListener struct{} + +func (*MockDataListener) Process(configType *config_center.ConfigChangeEvent) {} + +type KubernetesRegistryTestSuite struct { + suite.Suite + + currentPod v1.Pod + + registry *kubernetesRegistry +} + +func (s *KubernetesRegistryTestSuite) SetupTest() { + + t := s.T() + var err error + + regurl, err := common.NewURL("registry://127.0.0.1:443", common.WithParamsValue(constant.ROLE_KEY, strconv.Itoa(common.PROVIDER))) + if err != nil { + t.Fatal(err) + } + + mock, err := newMockKubernetesRegistry(®url, s.currentPod.GetNamespace(), func() (kubernetes.Interface, error) { + + out := fake.NewSimpleClientset() + + // mock current pod + if _, err := out.CoreV1().Pods(s.currentPod.GetNamespace()).Create(&s.currentPod); err != nil { + t.Fatal(err) + } + return out, nil + }) + if err != nil { + t.Fatal(err) + } + + s.registry = mock.(*kubernetesRegistry) +} + +func (s *KubernetesRegistryTestSuite) SetupSuite() { + + t := s.T() + + const ( + // kubernetes inject the var + podNameKey = "HOSTNAME" + nameSpaceKey = "NAMESPACE" + ) + + // 1. install test data + if err := json.Unmarshal([]byte(clientPodJsonData), &s.currentPod); err != nil { + t.Fatal(err) + } + + // 2. set downward-api inject env + if err := os.Setenv(podNameKey, s.currentPod.GetName()); err != nil { + t.Fatal(err) + } + if err := os.Setenv(nameSpaceKey, s.currentPod.GetNamespace()); err != nil { + t.Fatal(err) + } +} + +// stop etcd server +func (s *KubernetesRegistryTestSuite) TearDownSuite() {} + +func (s *KubernetesRegistryTestSuite) TestDataChange() { + + t := s.T() + + listener := NewRegistryDataListener(&MockDataListener{}) + url, _ := common.NewURL("jsonrpc%3A%2F%2F127.0.0.1%3A20001%2Fcom.ikurento.user.UserProvider%3Fanyhost%3Dtrue%26app.version%3D0.0.1%26application%3DBDTService%26category%3Dproviders%26cluster%3Dfailover%26dubbo%3Ddubbo-provider-golang-2.6.0%26environment%3Ddev%26group%3D%26interface%3Dcom.ikurento.user.UserProvider%26ip%3D10.32.20.124%26loadbalance%3Drandom%26methods.GetUser.loadbalance%3Drandom%26methods.GetUser.retries%3D1%26methods.GetUser.weight%3D0%26module%3Ddubbogo%2Buser-info%2Bserver%26name%3DBDTService%26organization%3Dikurento.com%26owner%3DZX%26pid%3D74500%26retries%3D0%26service.filter%3Decho%26side%3Dprovider%26timestamp%3D1560155407%26version%3D%26warmup%3D100") + listener.AddInterestedURL(&url) + if !listener.DataChange(remoting.Event{Path: "/dubbo/com.ikurento.user.UserProvider/providers/jsonrpc%3A%2F%2F127.0.0.1%3A20001%2Fcom.ikurento.user.UserProvider%3Fanyhost%3Dtrue%26app.version%3D0.0.1%26application%3DBDTService%26category%3Dproviders%26cluster%3Dfailover%26dubbo%3Ddubbo-provider-golang-2.6.0%26environment%3Ddev%26group%3D%26interface%3Dcom.ikurento.user.UserProvider%26ip%3D10.32.20.124%26loadbalance%3Drandom%26methods.GetUser.loadbalance%3Drandom%26methods.GetUser.retries%3D1%26methods.GetUser.weight%3D0%26module%3Ddubbogo%2Buser-info%2Bserver%26name%3DBDTService%26organization%3Dikurento.com%26owner%3DZX%26pid%3D74500%26retries%3D0%26service.filter%3Decho%26side%3Dprovider%26timestamp%3D1560155407%26version%3D%26warmup%3D100"}) { + t.Fatal("data change not ok") + } } -func (*MockDataListener) Process(configType *config_center.ConfigChangeEvent) { +func TestKubernetesRegistrySuite(t *testing.T) { + suite.Run(t, &KubernetesRegistryTestSuite{}) } diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index c68f0963a9..8ebe688290 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -28,6 +28,7 @@ import ( import ( "github.com/dubbogo/gost/net" perrors "github.com/pkg/errors" + k8s "k8s.io/client-go/kubernetes" ) import ( @@ -160,3 +161,23 @@ func newKubernetesRegistry(url *common.URL) (registry.Registry, error) { return r, nil } + +func newMockKubernetesRegistry(url *common.URL, namespace string, clientGeneratorFunc func() (k8s.Interface, error)) ( + registry.Registry, + error, +) { + + var err error + + r := &kubernetesRegistry{} + + r.InitBaseRegistry(url, r) + r.client, err = kubernetes.NewMockClient(namespace, clientGeneratorFunc) + if err != nil { + return nil, perrors.WithMessage(err, "new mock client") + } + r.WaitGroup().Add(1) //zk client start successful, then wg +1 + go kubernetes.HandleClientRestart(r) + r.InitListeners() + return r, nil +} diff --git a/registry/kubernetes/registry_test.go b/registry/kubernetes/registry_test.go index 134217f3c8..2bd744d92b 100644 --- a/registry/kubernetes/registry_test.go +++ b/registry/kubernetes/registry_test.go @@ -18,19 +18,86 @@ package kubernetes import ( - "testing" + "time" ) -func Test_Register(t *testing.T) { +import ( + "github.com/stretchr/testify/assert" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/common/constant" +) + +func (s *KubernetesRegistryTestSuite) TestRegister() { + t := s.T() + + url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) + + err := s.registry.Register(url) + assert.NoError(t, err) + _, _, err = s.registry.client.GetChildren("/dubbo/com.ikurento.user.UserProvider/providers") + if err != nil { + t.Fatal(err) + } } -func Test_Subscribe(t *testing.T) { +func (s *KubernetesRegistryTestSuite) TestSubscribe() { + + t := s.T() + + url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) + + listener, err := s.registry.DoSubscribe(&url) + if err != nil { + t.Fatal(err) + } + + go func() { + err := s.registry.Register(url) + if err != nil { + t.Fatal(err) + } + }() + + serviceEvent, err := listener.Next() + if err != nil { + t.Fatal(err) + } + + t.Logf("got event %s", serviceEvent) } -func Test_ConsumerDestory(t *testing.T) { +func (s *KubernetesRegistryTestSuite) TestConsumerDestroy() { + + t := s.T() + url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) + + _, err := s.registry.DoSubscribe(&url) + if err != nil { + t.Fatal(err) + } + + //listener.Close() + time.Sleep(1e9) + s.registry.Destroy() + + assert.Equal(t, false, s.registry.IsAvailable()) } -func Test_ProviderDestory(t *testing.T) { +func (s *KubernetesRegistryTestSuite) TestProviderDestroy() { + + t := s.T() + + url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) + err := s.registry.Register(url) + assert.NoError(t, err) + + //listener.Close() + time.Sleep(1e9) + s.registry.Destroy() + assert.Equal(t, false, s.registry.IsAvailable()) } diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index dd1c0a3911..5bf5776625 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -105,8 +105,14 @@ func getCurrentNameSpace() (string, error) { return v, nil } -// new mock client -// new a client for test +// NewMockClient +// export for registry package test +func NewMockClient(namespace string, mockClientGenerator func() (kubernetes.Interface, error)) (*Client, error) { + return newMockClient(namespace, mockClientGenerator) +} + +// newMockClient +// new a client for test func newMockClient(namespace string, mockClientGenerator func() (kubernetes.Interface, error)) (*Client, error) { rawClient, err := mockClientGenerator() diff --git a/remoting/kubernetes/client_test.go b/remoting/kubernetes/client_test.go index ffd3540402..846745bb62 100644 --- a/remoting/kubernetes/client_test.go +++ b/remoting/kubernetes/client_test.go @@ -27,7 +27,6 @@ import ( ) import ( - "github.com/pkg/errors" "github.com/stretchr/testify/suite" v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" @@ -194,303 +193,11 @@ var clientPodJsonData = `{ } ` -var server1PodJsonData = `{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "dubbo.io/annotation": "W3siayI6Ii9kdWJibyIsInYiOiIifSx7ImsiOiIvZHViYm8vY29tLmlrdXJlbnRvLnVzZXIuVXNlclByb3ZpZGVyIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvcHJvdmlkZXJzIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvcHJvdmlkZXJzL2R1YmJvJTNBJTJGJTJGMTcyLjE3LjAuNyUzQTIwMDAwJTJGVXNlclByb3ZpZGVyJTNGYWNjZXNzbG9nJTNEJTI2YW55aG9zdCUzRHRydWUlMjZhcHAudmVyc2lvbiUzRDAuMC4xJTI2YXBwbGljYXRpb24lM0RCRFRTZXJ2aWNlJTI2YmVhbi5uYW1lJTNEVXNlclByb3ZpZGVyJTI2Y2F0ZWdvcnklM0Rwcm92aWRlcnMlMjZjbHVzdGVyJTNEZmFpbG92ZXIlMjZkdWJibyUzRGR1YmJvLXByb3ZpZGVyLWdvbGFuZy0yLjYuMCUyNmVudmlyb25tZW50JTNEZGV2JTI2ZXhlY3V0ZS5saW1pdCUzRCUyNmV4ZWN1dGUubGltaXQucmVqZWN0ZWQuaGFuZGxlciUzRCUyNmdyb3VwJTNEJTI2aW50ZXJmYWNlJTNEY29tLmlrdXJlbnRvLnVzZXIuVXNlclByb3ZpZGVyJTI2aXAlM0QxNzIuMTcuMC43JTI2bG9hZGJhbGFuY2UlM0RyYW5kb20lMjZtZXRob2RzLkdldFVzZXIubG9hZGJhbGFuY2UlM0RyYW5kb20lMjZtZXRob2RzLkdldFVzZXIucmV0cmllcyUzRDElMjZtZXRob2RzLkdldFVzZXIudHBzLmxpbWl0LmludGVydmFsJTNEJTI2bWV0aG9kcy5HZXRVc2VyLnRwcy5saW1pdC5yYXRlJTNEJTI2bWV0aG9kcy5HZXRVc2VyLnRwcy5saW1pdC5zdHJhdGVneSUzRCUyNm1ldGhvZHMuR2V0VXNlci53ZWlnaHQlM0QwJTI2bW9kdWxlJTNEZHViYm9nbyUyQnVzZXItaW5mbyUyQnNlcnZlciUyNm5hbWUlM0RCRFRTZXJ2aWNlJTI2b3JnYW5pemF0aW9uJTNEaWt1cmVudG8uY29tJTI2b3duZXIlM0RaWCUyNnBpZCUzRDEwJTI2cmVnaXN0cnkucm9sZSUzRDMlMjZyZXRyaWVzJTNEJTI2c2VydmljZS5maWx0ZXIlM0RlY2hvJTI1MkN0b2tlbiUyNTJDYWNjZXNzbG9nJTI1MkN0cHMlMjUyQ2V4ZWN1dGUlMjZzaWRlJTNEcHJvdmlkZXIlMjZ0aW1lc3RhbXAlM0QxNTg0MDcwODEwJTI2dHBzLmxpbWl0LmludGVydmFsJTNEJTI2dHBzLmxpbWl0LnJhdGUlM0QlMjZ0cHMubGltaXQucmVqZWN0ZWQuaGFuZGxlciUzRCUyNnRwcy5saW1pdC5zdHJhdGVneSUzRCUyNnRwcy5saW1pdGVyJTNEJTI2dmVyc2lvbiUzRCUyNndhcm11cCUzRDEwMCIsInYiOiIifV0=" - }, - "creationTimestamp": "2020-03-13T03:38:57Z", - "generateName": "server-5b8f9f85c6-", - "labels": { - "dubbo.io/label": "dubbo.io-value", - "pod-template-hash": "5b8f9f85c6", - "role": "server" - }, - "name": "server-5b8f9f85c6-2w5rq", - "namespace": "default", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "server-5b8f9f85c6", - "uid": "65e9d2b0-f286-4b21-ac31-260f1412556b" - } - ], - "resourceVersion": "2449678", - "selfLink": "/api/v1/namespaces/default/pods/server-5b8f9f85c6-2w5rq", - "uid": "ae7497c7-396d-40c5-b53e-5720a696e4ee" - }, - "spec": { - "containers": [ - { - "env": [ - { - "name": "NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - } - ], - "image": "registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-server", - "imagePullPolicy": "Always", - "name": "server", - "resources": {}, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "dubbo-sa-token-l2lzh", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "minikube", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "dubbo-sa", - "serviceAccountName": "dubbo-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "dubbo-sa-token-l2lzh", - "secret": { - "defaultMode": 420, - "secretName": "dubbo-sa-token-l2lzh" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2020-03-13T03:38:57Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2020-03-13T03:40:10Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2020-03-13T03:40:10Z", - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2020-03-13T03:38:57Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://88144bc6eabf783c0954c2e078a7270c8a0246d6f8af081dfcc0956e8c3cd2de", - "image": "registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-server:latest", - "imageID": "docker-pullable://registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-server@sha256:60654ddba3a16ca3de52c8e30650a4c1d8b2ed8f8542af489b7a5a459e46fe6b", - "lastState": {}, - "name": "server", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2020-03-13T03:40:10Z" - } - } - } - ], - "hostIP": "10.0.2.15", - "phase": "Running", - "podIP": "172.17.0.7", - "qosClass": "BestEffort", - "startTime": "2020-03-13T03:38:57Z" - } -} -` - -var server2PodJsonData = `{ - "apiVersion": "v1", - "kind": "Pod", - "metadata": { - "annotations": { - "dubbo.io/annotation": "W3siayI6Ii9kdWJibyIsInYiOiIifSx7ImsiOiIvZHViYm8vY29tLmlrdXJlbnRvLnVzZXIuVXNlclByb3ZpZGVyIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvcHJvdmlkZXJzIiwidiI6IiJ9LHsiayI6Ii9kdWJiby9jb20uaWt1cmVudG8udXNlci5Vc2VyUHJvdmlkZXIvcHJvdmlkZXJzL2R1YmJvJTNBJTJGJTJGMTcyLjE3LjAuNiUzQTIwMDAwJTJGVXNlclByb3ZpZGVyJTNGYWNjZXNzbG9nJTNEJTI2YW55aG9zdCUzRHRydWUlMjZhcHAudmVyc2lvbiUzRDAuMC4xJTI2YXBwbGljYXRpb24lM0RCRFRTZXJ2aWNlJTI2YmVhbi5uYW1lJTNEVXNlclByb3ZpZGVyJTI2Y2F0ZWdvcnklM0Rwcm92aWRlcnMlMjZjbHVzdGVyJTNEZmFpbG92ZXIlMjZkdWJibyUzRGR1YmJvLXByb3ZpZGVyLWdvbGFuZy0yLjYuMCUyNmVudmlyb25tZW50JTNEZGV2JTI2ZXhlY3V0ZS5saW1pdCUzRCUyNmV4ZWN1dGUubGltaXQucmVqZWN0ZWQuaGFuZGxlciUzRCUyNmdyb3VwJTNEJTI2aW50ZXJmYWNlJTNEY29tLmlrdXJlbnRvLnVzZXIuVXNlclByb3ZpZGVyJTI2aXAlM0QxNzIuMTcuMC42JTI2bG9hZGJhbGFuY2UlM0RyYW5kb20lMjZtZXRob2RzLkdldFVzZXIubG9hZGJhbGFuY2UlM0RyYW5kb20lMjZtZXRob2RzLkdldFVzZXIucmV0cmllcyUzRDElMjZtZXRob2RzLkdldFVzZXIudHBzLmxpbWl0LmludGVydmFsJTNEJTI2bWV0aG9kcy5HZXRVc2VyLnRwcy5saW1pdC5yYXRlJTNEJTI2bWV0aG9kcy5HZXRVc2VyLnRwcy5saW1pdC5zdHJhdGVneSUzRCUyNm1ldGhvZHMuR2V0VXNlci53ZWlnaHQlM0QwJTI2bW9kdWxlJTNEZHViYm9nbyUyQnVzZXItaW5mbyUyQnNlcnZlciUyNm5hbWUlM0RCRFRTZXJ2aWNlJTI2b3JnYW5pemF0aW9uJTNEaWt1cmVudG8uY29tJTI2b3duZXIlM0RaWCUyNnBpZCUzRDEwJTI2cmVnaXN0cnkucm9sZSUzRDMlMjZyZXRyaWVzJTNEJTI2c2VydmljZS5maWx0ZXIlM0RlY2hvJTI1MkN0b2tlbiUyNTJDYWNjZXNzbG9nJTI1MkN0cHMlMjUyQ2V4ZWN1dGUlMjZzaWRlJTNEcHJvdmlkZXIlMjZ0aW1lc3RhbXAlM0QxNTg0MDcwODA5JTI2dHBzLmxpbWl0LmludGVydmFsJTNEJTI2dHBzLmxpbWl0LnJhdGUlM0QlMjZ0cHMubGltaXQucmVqZWN0ZWQuaGFuZGxlciUzRCUyNnRwcy5saW1pdC5zdHJhdGVneSUzRCUyNnRwcy5saW1pdGVyJTNEJTI2dmVyc2lvbiUzRCUyNndhcm11cCUzRDEwMCIsInYiOiIifV0=" - }, - "creationTimestamp": "2020-03-13T03:38:57Z", - "generateName": "server-5b8f9f85c6-", - "labels": { - "dubbo.io/label": "dubbo.io-value", - "pod-template-hash": "5b8f9f85c6", - "role": "server" - }, - "name": "server-5b8f9f85c6-xk5md", - "namespace": "default", - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "blockOwnerDeletion": true, - "controller": true, - "kind": "ReplicaSet", - "name": "server-5b8f9f85c6", - "uid": "65e9d2b0-f286-4b21-ac31-260f1412556b" - } - ], - "resourceVersion": "2449667", - "selfLink": "/api/v1/namespaces/default/pods/server-5b8f9f85c6-xk5md", - "uid": "9e59e164-6620-473b-a983-472ebc1120e9" - }, - "spec": { - "containers": [ - { - "env": [ - { - "name": "NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - } - ], - "image": "registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-server", - "imagePullPolicy": "Always", - "name": "server", - "resources": {}, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "volumeMounts": [ - { - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", - "name": "dubbo-sa-token-l2lzh", - "readOnly": true - } - ] - } - ], - "dnsPolicy": "ClusterFirst", - "enableServiceLinks": true, - "nodeName": "minikube", - "priority": 0, - "restartPolicy": "Always", - "schedulerName": "default-scheduler", - "securityContext": {}, - "serviceAccount": "dubbo-sa", - "serviceAccountName": "dubbo-sa", - "terminationGracePeriodSeconds": 30, - "tolerations": [ - { - "effect": "NoExecute", - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "tolerationSeconds": 300 - }, - { - "effect": "NoExecute", - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "tolerationSeconds": 300 - } - ], - "volumes": [ - { - "name": "dubbo-sa-token-l2lzh", - "secret": { - "defaultMode": 420, - "secretName": "dubbo-sa-token-l2lzh" - } - } - ] - }, - "status": { - "conditions": [ - { - "lastProbeTime": null, - "lastTransitionTime": "2020-03-13T03:38:57Z", - "status": "True", - "type": "Initialized" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2020-03-13T03:40:09Z", - "status": "True", - "type": "Ready" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2020-03-13T03:40:09Z", - "status": "True", - "type": "ContainersReady" - }, - { - "lastProbeTime": null, - "lastTransitionTime": "2020-03-13T03:38:57Z", - "status": "True", - "type": "PodScheduled" - } - ], - "containerStatuses": [ - { - "containerID": "docker://442dc055392cc720b6a6eb0bc4105f13ea86e63cbdced5c83f15463bc61add76", - "image": "registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-server:latest", - "imageID": "docker-pullable://registry.cn-hangzhou.aliyuncs.com/scottwang/dubbogo-server@sha256:60654ddba3a16ca3de52c8e30650a4c1d8b2ed8f8542af489b7a5a459e46fe6b", - "lastState": {}, - "name": "server", - "ready": true, - "restartCount": 0, - "state": { - "running": { - "startedAt": "2020-03-13T03:40:09Z" - } - } - } - ], - "hostIP": "10.0.2.15", - "phase": "Running", - "podIP": "172.17.0.6", - "qosClass": "BestEffort", - "startTime": "2020-03-13T03:38:57Z" - } -}` - type KubernetesClientTestSuite struct { suite.Suite - client *Client - - currentPod v1.Pod - fakeServerPod1 v1.Pod - fakeServerPod2 v1.Pod + client *Client + currentPod v1.Pod } func (s *KubernetesClientTestSuite) SetupSuite() { @@ -501,12 +208,6 @@ func (s *KubernetesClientTestSuite) SetupSuite() { if err := json.Unmarshal([]byte(clientPodJsonData), &s.currentPod); err != nil { t.Fatal(err) } - if err := json.Unmarshal([]byte(server1PodJsonData), &s.fakeServerPod1); err != nil { - t.Fatal(err) - } - if err := json.Unmarshal([]byte(server2PodJsonData), &s.fakeServerPod2); err != nil { - t.Fatal(err) - } // 2. set downward-api inject env if err := os.Setenv(podNameKey, s.currentPod.GetName()); err != nil { @@ -519,8 +220,6 @@ func (s *KubernetesClientTestSuite) SetupSuite() { func (s *KubernetesClientTestSuite) TearDownSuite() { s.client.Close() - os.Unsetenv(podNameKey) - os.Unsetenv(nameSpaceKey) } func (s *KubernetesClientTestSuite) SetupTest() { @@ -533,7 +232,7 @@ func (s *KubernetesClientTestSuite) SetupTest() { // mock current pod if _, err := out.CoreV1().Pods(s.currentPod.GetNamespace()).Create(&s.currentPod); err != nil { - return nil, errors.WithMessage(err, "mock current pod ") + t.Fatal(err) } return out, nil }) From f29c788c9d0824f7978326cb6a5e4e6d6cba756a Mon Sep 17 00:00:00 2001 From: scott Date: Sat, 14 Mar 2020 12:22:20 +0800 Subject: [PATCH 18/61] Fix ci client close race condition --- registry/kubernetes/listener_test.go | 10 +--- registry/kubernetes/registry_test.go | 31 ++++++---- remoting/kubernetes/client_test.go | 86 +++++++++++++++------------- 3 files changed, 69 insertions(+), 58 deletions(-) diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go index 16bbbf8c7a..c9ff626088 100644 --- a/registry/kubernetes/listener_test.go +++ b/registry/kubernetes/listener_test.go @@ -187,14 +187,11 @@ type KubernetesRegistryTestSuite struct { suite.Suite currentPod v1.Pod - - registry *kubernetesRegistry } -func (s *KubernetesRegistryTestSuite) SetupTest() { +func (s *KubernetesRegistryTestSuite) initRegistry() *kubernetesRegistry { t := s.T() - var err error regurl, err := common.NewURL("registry://127.0.0.1:443", common.WithParamsValue(constant.ROLE_KEY, strconv.Itoa(common.PROVIDER))) if err != nil { @@ -215,7 +212,7 @@ func (s *KubernetesRegistryTestSuite) SetupTest() { t.Fatal(err) } - s.registry = mock.(*kubernetesRegistry) + return mock.(*kubernetesRegistry) } func (s *KubernetesRegistryTestSuite) SetupSuite() { @@ -242,9 +239,6 @@ func (s *KubernetesRegistryTestSuite) SetupSuite() { } } -// stop etcd server -func (s *KubernetesRegistryTestSuite) TearDownSuite() {} - func (s *KubernetesRegistryTestSuite) TestDataChange() { t := s.T() diff --git a/registry/kubernetes/registry_test.go b/registry/kubernetes/registry_test.go index 2bd744d92b..cc5ccbb149 100644 --- a/registry/kubernetes/registry_test.go +++ b/registry/kubernetes/registry_test.go @@ -34,11 +34,14 @@ func (s *KubernetesRegistryTestSuite) TestRegister() { t := s.T() + r := s.initRegistry() + defer r.Destroy() + url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) - err := s.registry.Register(url) + err := r.Register(url) assert.NoError(t, err) - _, _, err = s.registry.client.GetChildren("/dubbo/com.ikurento.user.UserProvider/providers") + _, _, err = r.client.GetChildren("/dubbo/com.ikurento.user.UserProvider/providers") if err != nil { t.Fatal(err) } @@ -48,15 +51,18 @@ func (s *KubernetesRegistryTestSuite) TestSubscribe() { t := s.T() + r := s.initRegistry() + defer r.Destroy() + url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) - listener, err := s.registry.DoSubscribe(&url) + listener, err := r.DoSubscribe(&url) if err != nil { t.Fatal(err) } go func() { - err := s.registry.Register(url) + err := r.Register(url) if err != nil { t.Fatal(err) } @@ -73,18 +79,21 @@ func (s *KubernetesRegistryTestSuite) TestSubscribe() { func (s *KubernetesRegistryTestSuite) TestConsumerDestroy() { t := s.T() + + r := s.initRegistry() + url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) - _, err := s.registry.DoSubscribe(&url) + _, err := r.DoSubscribe(&url) if err != nil { t.Fatal(err) } //listener.Close() time.Sleep(1e9) - s.registry.Destroy() + r.Destroy() - assert.Equal(t, false, s.registry.IsAvailable()) + assert.Equal(t, false, r.IsAvailable()) } @@ -92,12 +101,14 @@ func (s *KubernetesRegistryTestSuite) TestProviderDestroy() { t := s.T() + r := s.initRegistry() + url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) - err := s.registry.Register(url) + err := r.Register(url) assert.NoError(t, err) //listener.Close() time.Sleep(1e9) - s.registry.Destroy() - assert.Equal(t, false, s.registry.IsAvailable()) + r.Destroy() + assert.Equal(t, false, r.IsAvailable()) } diff --git a/remoting/kubernetes/client_test.go b/remoting/kubernetes/client_test.go index 846745bb62..c4a4e822ec 100644 --- a/remoting/kubernetes/client_test.go +++ b/remoting/kubernetes/client_test.go @@ -24,10 +24,9 @@ import ( "sync" "testing" "time" -) -import ( "github.com/stretchr/testify/suite" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" @@ -196,10 +195,29 @@ var clientPodJsonData = `{ type KubernetesClientTestSuite struct { suite.Suite - client *Client currentPod v1.Pod } +func (s *KubernetesClientTestSuite) initClient() *Client { + + t := s.T() + + client, err := newMockClient(s.currentPod.GetNamespace(), func() (kubernetes.Interface, error) { + + out := fake.NewSimpleClientset() + + // mock current pod + if _, err := out.CoreV1().Pods(s.currentPod.GetNamespace()).Create(&s.currentPod); err != nil { + t.Fatal(err) + } + return out, nil + }) + if err != nil { + t.Fatal(err) + } + return client +} + func (s *KubernetesClientTestSuite) SetupSuite() { t := s.T() @@ -218,39 +236,19 @@ func (s *KubernetesClientTestSuite) SetupSuite() { } } -func (s *KubernetesClientTestSuite) TearDownSuite() { - s.client.Close() -} - -func (s *KubernetesClientTestSuite) SetupTest() { - - t := s.T() - var err error - s.client, err = newMockClient(s.currentPod.GetNamespace(), func() (kubernetes.Interface, error) { - - out := fake.NewSimpleClientset() - - // mock current pod - if _, err := out.CoreV1().Pods(s.currentPod.GetNamespace()).Create(&s.currentPod); err != nil { - t.Fatal(err) - } - return out, nil - }) - if err != nil { - t.Fatal(err) - } -} - func (s *KubernetesClientTestSuite) TestClientValid() { t := s.T() - if s.client.Valid() != true { + client := s.initClient() + defer client.Close() + + if client.Valid() != true { t.Fatal("client is not valid") } - s.client.Close() - if s.client.Valid() != false { + client.Close() + if client.Valid() != false { t.Fatal("client is valid") } } @@ -259,14 +257,16 @@ func (s *KubernetesClientTestSuite) TestClientDone() { t := s.T() + client := s.initClient() + go func() { time.Sleep(time.Second) - s.client.Close() + client.Close() }() - <-s.client.Done() + <-client.Done() - if s.client.Valid() == true { + if client.Valid() == true { t.Fatal("client should be invalid then") } } @@ -274,14 +274,16 @@ func (s *KubernetesClientTestSuite) TestClientDone() { func (s *KubernetesClientTestSuite) TestClientCreateKV() { t := s.T() - defer s.client.Close() + + client := s.initClient() + defer client.Close() for _, tc := range tests { k := tc.input.k v := tc.input.v - if err := s.client.Create(k, v); err != nil { + if err := client.Create(k, v); err != nil { t.Fatal(err) } @@ -291,7 +293,9 @@ func (s *KubernetesClientTestSuite) TestClientCreateKV() { func (s *KubernetesClientTestSuite) TestClientGetChildrenKVList() { t := s.T() - defer s.client.Close() + + client := s.initClient() + defer client.Close() expect := make(map[string]string) got := make(map[string]string) @@ -305,12 +309,12 @@ func (s *KubernetesClientTestSuite) TestClientGetChildrenKVList() { expect[k] = v } - if err := s.client.Create(k, v); err != nil { + if err := client.Create(k, v); err != nil { t.Fatal(err) } } - kList, vList, err := s.client.GetChildren(prefix) + kList, vList, err := client.GetChildren(prefix) if err != nil { t.Fatal(err) } @@ -332,6 +336,8 @@ func (s *KubernetesClientTestSuite) TestClientWatch() { t := s.T() + client := s.initClient() + wg := sync.WaitGroup{} wg.Add(1) @@ -339,7 +345,7 @@ func (s *KubernetesClientTestSuite) TestClientWatch() { defer wg.Done() - wc, err := s.client.WatchWithPrefix(prefix) + wc, err := client.WatchWithPrefix(prefix) if err != nil { t.Fatal(err) } @@ -355,12 +361,12 @@ func (s *KubernetesClientTestSuite) TestClientWatch() { k := tc.input.k v := tc.input.v - if err := s.client.Create(k, v); err != nil { + if err := client.Create(k, v); err != nil { t.Fatal(err) } } - s.client.Close() + client.Close() wg.Wait() } From f11f6fb6578e7a1acbbd06ea79963a61ff7575a5 Mon Sep 17 00:00:00 2001 From: scott Date: Sat, 14 Mar 2020 12:37:59 +0800 Subject: [PATCH 19/61] Fix remote/kubernetes unit-test race condition --- go.sum | 6 ++++++ remoting/kubernetes/client_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/go.sum b/go.sum index eb775ec08a..c2be2b1e43 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,7 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v0.0.0-20180123065059-ebf56d35bba7 h1:bGT+Ub6bpzHl7AAYQhBrZ5nYTAH2SF/848WducU0Ao4= github.com/fatih/structs v0.0.0-20180123065059-ebf56d35bba7/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -288,6 +289,7 @@ github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443 h1:O/pT5C1Q3mVXMyu github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443/go.mod h1:bEpDU35nTu0ey1EXjwNwPjI9xErAsoOCmcMb9GKvyxo= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -380,9 +382,11 @@ github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2/go.mod h github.com/oklog/run v0.0.0-20180308005104-6934b124db28 h1:Hbr3fbVPXea52oPQeP7KLSxP52g6SFaNY1IqAmUyEW0= github.com/oklog/run v0.0.0-20180308005104-6934b124db28/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -574,6 +578,7 @@ gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUy gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -587,6 +592,7 @@ gopkg.in/ory-am/dockertest.v3 v3.3.4/go.mod h1:s9mmoLkaGeAh97qygnNj4xWkiN7e1SKek gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/remoting/kubernetes/client_test.go b/remoting/kubernetes/client_test.go index c4a4e822ec..08bc96794a 100644 --- a/remoting/kubernetes/client_test.go +++ b/remoting/kubernetes/client_test.go @@ -297,6 +297,27 @@ func (s *KubernetesClientTestSuite) TestClientGetChildrenKVList() { client := s.initClient() defer client.Close() + wg := sync.WaitGroup{} + wg.Add(1) + + go func() { + defer wg.Done() + + wc, err := client.WatchWithPrefix(prefix) + if err != nil { + t.Fatal(err) + } + i := 0 + for e := range wc { + i++ + t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) + if i == 3 { + // already sync all event + return + } + } + }() + expect := make(map[string]string) got := make(map[string]string) @@ -314,6 +335,11 @@ func (s *KubernetesClientTestSuite) TestClientGetChildrenKVList() { } } + // must wait client sync all create event + + wg.Wait() + + // start get all children kList, vList, err := client.GetChildren(prefix) if err != nil { t.Fatal(err) From e5c3ff501a7628292dae58703935fcac766abb89 Mon Sep 17 00:00:00 2001 From: scott Date: Sat, 14 Mar 2020 21:00:32 +0800 Subject: [PATCH 20/61] Fix nil init --- registry/kubernetes/listener.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registry/kubernetes/listener.go b/registry/kubernetes/listener.go index 1f4f2ed215..cf254db8ed 100644 --- a/registry/kubernetes/listener.go +++ b/registry/kubernetes/listener.go @@ -40,7 +40,7 @@ type dataListener struct { // NewRegistryDataListener ... func NewRegistryDataListener(listener config_center.ConfigurationListener) *dataListener { - return &dataListener{listener: listener, interestedURL: []*common.URL{}} + return &dataListener{listener: listener} } func (l *dataListener) AddInterestedURL(url *common.URL) { From 9af6c5394facba7283576bfe86ee3883a559f3d1 Mon Sep 17 00:00:00 2001 From: scott Date: Sat, 14 Mar 2020 21:26:26 +0800 Subject: [PATCH 21/61] Fix latest comment --- registry/kubernetes/listener.go | 2 +- registry/kubernetes/registry.go | 11 ++++++----- remoting/kubernetes/client.go | 16 ++++++++-------- remoting/kubernetes/store.go | 6 ++---- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/registry/kubernetes/listener.go b/registry/kubernetes/listener.go index cf254db8ed..6ddcd4f07d 100644 --- a/registry/kubernetes/listener.go +++ b/registry/kubernetes/listener.go @@ -92,7 +92,7 @@ func (l *configurationListener) Next() (*registry.ServiceEvent, error) { for { select { case <-l.registry.Done(): - logger.Warnf("listener's kubernetes client connection is broken, so kubernetes event listener exit now.") + logger.Warnf("listener's kubernetes client connection is broken, so kubernetes event listener exits now.") return nil, perrors.New("listener stopped") case e := <-l.events: diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index 8ebe688290..2a63660cc0 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -150,7 +150,7 @@ func newKubernetesRegistry(url *common.URL) (registry.Registry, error) { r.InitBaseRegistry(url, r) if err := kubernetes.ValidateClient(r); err != nil { - return nil, err + return nil, perrors.WithStack(err) } r.WaitGroup().Add(1) @@ -162,10 +162,11 @@ func newKubernetesRegistry(url *common.URL) (registry.Registry, error) { return r, nil } -func newMockKubernetesRegistry(url *common.URL, namespace string, clientGeneratorFunc func() (k8s.Interface, error)) ( - registry.Registry, - error, -) { +func newMockKubernetesRegistry( + url *common.URL, + namespace string, + clientGeneratorFunc func() (k8s.Interface, error), +) (registry.Registry, error) { var err error diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index 5bf5776625..2091cc2f1e 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -418,7 +418,6 @@ func (c *Client) handleWatchedPodEvent(p *v1.Pod, eventType watch.EventType) { func (c *Client) unmarshalRecord(record string) ([]*Object, error) { if len(record) == 0 { - // NOTICE: // []*Object is nil. return nil, nil } @@ -503,12 +502,14 @@ func (c *Client) patchCurrentPod(patch []byte) (*v1.Pod, error) { return updatedPod, nil } -// assemble the dubbo kubernete label +// assemble the dubbo kubernetes label // every dubbo instance should be labeled spec {"dubbo.io/label":"dubbo.io/label-value"} label -func (c *Client) assembleDUBBOLabel(currentPod *v1.Pod) (oldPod *v1.Pod, newPod *v1.Pod, err error) { +func (c *Client) assembleDUBBOLabel(currentPod *v1.Pod) (*v1.Pod, *v1.Pod, error) { - oldPod = &v1.Pod{} - newPod = &v1.Pod{} + var ( + oldPod = &v1.Pod{} + newPod = &v1.Pod{} + ) oldPod.Labels = make(map[string]string, 8) newPod.Labels = make(map[string]string, 8) @@ -517,8 +518,7 @@ func (c *Client) assembleDUBBOLabel(currentPod *v1.Pod) (oldPod *v1.Pod, newPod if currentPod.GetLabels()[DubboIOLabelKey] == DubboIOLabelValue { // already have label - err = ErrDubboLabelAlreadyExist - return + return nil, nil, ErrDubboLabelAlreadyExist } } @@ -529,7 +529,7 @@ func (c *Client) assembleDUBBOLabel(currentPod *v1.Pod) (oldPod *v1.Pod, newPod } // assign new label for current pod newPod.Labels[DubboIOLabelKey] = DubboIOLabelValue - return + return oldPod, newPod, nil } // assemble the dubbo kubernetes annotations diff --git a/remoting/kubernetes/store.go b/remoting/kubernetes/store.go index 5feab9f0b0..06f32cc0cb 100644 --- a/remoting/kubernetes/store.go +++ b/remoting/kubernetes/store.go @@ -84,7 +84,7 @@ type Store interface { Done() <-chan struct{} } -// Stopped Watcher +// Watcher type Watcher interface { // the watcher's id ID() string @@ -200,9 +200,7 @@ func (s *storeImpl) Put(object *Object) error { } // valid -// valid the client status -// NOTICE: -// should protected by lock +// valid the client status should protected by lock func (s *storeImpl) valid() error { select { case <-s.ctx.Done(): From ae667e98c3f518f61b764f017affb51815942eac Mon Sep 17 00:00:00 2001 From: scott Date: Sat, 14 Mar 2020 21:50:21 +0800 Subject: [PATCH 22/61] Fix latest alex comment --- registry/kubernetes/listener_test.go | 1 + remoting/kubernetes/listener.go | 6 +++--- remoting/kubernetes/store_test.go | 11 ++++------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go index c9ff626088..f89154e32e 100644 --- a/registry/kubernetes/listener_test.go +++ b/registry/kubernetes/listener_test.go @@ -237,6 +237,7 @@ func (s *KubernetesRegistryTestSuite) SetupSuite() { if err := os.Setenv(nameSpaceKey, s.currentPod.GetNamespace()); err != nil { t.Fatal(err) } + } func (s *KubernetesRegistryTestSuite) TestDataChange() { diff --git a/remoting/kubernetes/listener.go b/remoting/kubernetes/listener.go index c8cc10f36a..c87559e7f4 100644 --- a/remoting/kubernetes/listener.go +++ b/remoting/kubernetes/listener.go @@ -33,7 +33,7 @@ import ( type EventListener struct { client *Client - keyMapLock sync.Mutex + keyMapLock sync.RWMutex keyMap map[string]struct{} wg sync.WaitGroup } @@ -157,9 +157,9 @@ func timeSecondDuration(sec int) time.Duration { // --------> ListenServiceNodeEvent func (l *EventListener) ListenServiceEvent(key string, listener remoting.DataListener) { - l.keyMapLock.Lock() + l.keyMapLock.RLock() _, ok := l.keyMap[key] - l.keyMapLock.Unlock() + l.keyMapLock.RUnlock() if ok { logger.Warnf("kubernetes-store key %s has already been listened.", key) return diff --git a/remoting/kubernetes/store_test.go b/remoting/kubernetes/store_test.go index 2bc8b277a7..11036adb40 100644 --- a/remoting/kubernetes/store_test.go +++ b/remoting/kubernetes/store_test.go @@ -19,7 +19,6 @@ package kubernetes import ( "context" - "fmt" "strconv" "sync" "testing" @@ -43,11 +42,10 @@ func TestStore(t *testing.T) { defer wg.Done() w, err := s.Watch("key-1", false) if err != nil { - fmt.Println("watch spec result", err) - return + t.Fatal(err) } for e := range w.ResultChan() { - fmt.Printf("consumer %s got %s\n", w.ID(), e.Key) + t.Logf("consumer %s got %s\n", w.ID(), e.Key) } }() } @@ -59,11 +57,10 @@ func TestStore(t *testing.T) { defer wg.Done() w, err := s.Watch("key", true) if err != nil { - fmt.Println("watch prefix result", err) - return + t.Fatal(err) } for e := range w.ResultChan() { - fmt.Printf("prefix consumer %s got %s\n", w.ID(), e.Key) + t.Logf("prefix consumer %s got %s\n", w.ID(), e.Key) } }() } From 059f9b853956e0e98846fe0172a3f364eb0859ff Mon Sep 17 00:00:00 2001 From: scott Date: Sat, 14 Mar 2020 21:54:01 +0800 Subject: [PATCH 23/61] Add double check for RWMutex --- remoting/kubernetes/listener.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/remoting/kubernetes/listener.go b/remoting/kubernetes/listener.go index c87559e7f4..da6d835159 100644 --- a/remoting/kubernetes/listener.go +++ b/remoting/kubernetes/listener.go @@ -166,6 +166,12 @@ func (l *EventListener) ListenServiceEvent(key string, listener remoting.DataLis } l.keyMapLock.Lock() + // double check + if _, ok := l.keyMap[key]; ok { + // another goroutine already set it + l.keyMapLock.Unlock() + return + } l.keyMap[key] = struct{}{} l.keyMapLock.Unlock() From d7ae998f02cc04c68c507bb7666ce39e178324f4 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 14:25:23 +0800 Subject: [PATCH 24/61] Fix registry package unit test cover --- registry/kubernetes/registry_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/registry/kubernetes/registry_test.go b/registry/kubernetes/registry_test.go index cc5ccbb149..22873606b6 100644 --- a/registry/kubernetes/registry_test.go +++ b/registry/kubernetes/registry_test.go @@ -18,6 +18,7 @@ package kubernetes import ( + "strconv" "time" ) @@ -112,3 +113,17 @@ func (s *KubernetesRegistryTestSuite) TestProviderDestroy() { r.Destroy() assert.Equal(t, false, r.IsAvailable()) } + +func (s *KubernetesRegistryTestSuite) TestNewRegistry() { + + t := s.T() + + regUrl, err := common.NewURL("registry://127.0.0.1:443", common.WithParamsValue(constant.ROLE_KEY, strconv.Itoa(common.PROVIDER))) + if err != nil { + t.Fatal(err) + } + _, err = newKubernetesRegistry(®Url) + if err == nil { + t.Fatal("not in cluster, should be a err") + } +} From 3349096c9f22be9a2f8c68276f114fb86ebd468d Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 14:48:20 +0800 Subject: [PATCH 25/61] Add test cover for remote/kubernetes --- remoting/etcdv3/coverage.html | 970 +++++++++++++++++++++++++++ remoting/kubernetes/client_test.go | 77 ++- remoting/kubernetes/facade_test.go | 102 +++ remoting/kubernetes/listener_test.go | 68 ++ 4 files changed, 1216 insertions(+), 1 deletion(-) create mode 100644 remoting/etcdv3/coverage.html create mode 100644 remoting/kubernetes/facade_test.go diff --git a/remoting/etcdv3/coverage.html b/remoting/etcdv3/coverage.html new file mode 100644 index 0000000000..04a7ab97ca --- /dev/null +++ b/remoting/etcdv3/coverage.html @@ -0,0 +1,970 @@ + + + + + + + + +
+ +
+ not tracked + + not covered + covered + +
+
+
+ + + + + + + +
+ + + diff --git a/remoting/kubernetes/client_test.go b/remoting/kubernetes/client_test.go index 08bc96794a..7345ddfbd5 100644 --- a/remoting/kubernetes/client_test.go +++ b/remoting/kubernetes/client_test.go @@ -236,6 +236,32 @@ func (s *KubernetesClientTestSuite) SetupSuite() { } } +func (s *KubernetesClientTestSuite) TestReadCurrentPodName() { + t := s.T() + + n, err := getCurrentPodName() + if err != nil { + t.Fatal(err) + } + + if n != s.currentPod.GetName() { + t.Fatalf("expect %s but got %s", s.currentPod.GetName(), n) + } + +} +func (s *KubernetesClientTestSuite) TestReadCurrentNameSpace() { + t := s.T() + + ns, err := getCurrentNameSpace() + if err != nil { + t.Fatal(err) + } + + if ns != s.currentPod.GetNamespace() { + t.Fatalf("expect %s but got %s", s.currentPod.GetNamespace(), ns) + } + +} func (s *KubernetesClientTestSuite) TestClientValid() { t := s.T() @@ -358,7 +384,7 @@ func (s *KubernetesClientTestSuite) TestClientGetChildrenKVList() { } -func (s *KubernetesClientTestSuite) TestClientWatch() { +func (s *KubernetesClientTestSuite) TestClientWatchPrefix() { t := s.T() @@ -396,6 +422,55 @@ func (s *KubernetesClientTestSuite) TestClientWatch() { wg.Wait() } +func (s *KubernetesClientTestSuite) TestNewClient() { + + t := s.T() + + _, err := newClient(s.currentPod.GetNamespace()) + if err == nil { + t.Fatal("the out of cluster test should fail") + } + +} + +func (s *KubernetesClientTestSuite) TestClientWatch() { + + t := s.T() + + client := s.initClient() + + wg := sync.WaitGroup{} + wg.Add(1) + + go func() { + + defer wg.Done() + + wc, err := client.Watch(prefix) + if err != nil { + t.Fatal(err) + } + + for e := range wc { + t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) + } + + }() + + for _, tc := range tests { + + k := tc.input.k + v := tc.input.v + + if err := client.Create(k, v); err != nil { + t.Fatal(err) + } + } + + client.Close() + wg.Wait() +} + func TestKubernetesClient(t *testing.T) { suite.Run(t, new(KubernetesClientTestSuite)) } diff --git a/remoting/kubernetes/facade_test.go b/remoting/kubernetes/facade_test.go new file mode 100644 index 0000000000..cb9f92723f --- /dev/null +++ b/remoting/kubernetes/facade_test.go @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kubernetes + +import ( + "sync" +) +import ( + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" +) +import ( + "github.com/apache/dubbo-go/common" +) + +type mockFacade struct { + client *Client + cltLock sync.Mutex + wg sync.WaitGroup + URL *common.URL + done chan struct{} +} + +func (r *mockFacade) Client() *Client { + return r.client +} + +func (r *mockFacade) SetClient(client *Client) { + r.client = client +} + +func (r *mockFacade) ClientLock() *sync.Mutex { + return &r.cltLock +} + +func (r *mockFacade) WaitGroup() *sync.WaitGroup { + return &r.wg +} + +func (r *mockFacade) Done() chan struct{} { + return r.done +} + +func (r *mockFacade) GetUrl() common.URL { + return *r.URL +} + +func (r *mockFacade) Destroy() { + close(r.done) + r.wg.Wait() +} + +func (r *mockFacade) RestartCallBack() bool { + return true +} + +func (r *mockFacade) IsAvailable() bool { + return true +} + +func (s *KubernetesClientTestSuite) Test_Facade() { + + t := s.T() + + mockClient, err := newMockClient(s.currentPod.GetNamespace(), func() (kubernetes.Interface, error) { + + out := fake.NewSimpleClientset() + + // mock current pod + if _, err := out.CoreV1().Pods(s.currentPod.GetNamespace()).Create(&s.currentPod); err != nil { + t.Fatal(err) + } + return out, nil + }) + if err != nil { + t.Fatal(err) + } + + url, _ := common.NewURL("mock://127.0.0.1") + m := &mockFacade{ + client: mockClient, + URL: &url, + } + + go HandleClientRestart(m) + mockClient.Close() +} diff --git a/remoting/kubernetes/listener_test.go b/remoting/kubernetes/listener_test.go index 2b6883a076..a9446782a5 100644 --- a/remoting/kubernetes/listener_test.go +++ b/remoting/kubernetes/listener_test.go @@ -17,10 +17,40 @@ package kubernetes +import ( + "time" +) + +import ( + "github.com/stretchr/testify/assert" +) + import ( "github.com/apache/dubbo-go/remoting" ) +var changedData = ` + dubbo.consumer.request_timeout=3s + dubbo.consumer.connect_timeout=5s + dubbo.application.organization=ikurento.com + dubbo.application.name=BDTService + dubbo.application.module=dubbogo user-info server + dubbo.application.version=0.0.1 + dubbo.application.owner=ZX + dubbo.application.environment=dev + dubbo.registries.hangzhouzk.protocol=zookeeper + dubbo.registries.hangzhouzk.timeout=3s + dubbo.registries.hangzhouzk.address=127.0.0.1:2181 + dubbo.registries.shanghaizk.protocol=zookeeper + dubbo.registries.shanghaizk.timeout=3s + dubbo.registries.shanghaizk.address=127.0.0.1:2182 + dubbo.service.com.ikurento.user.UserProvider.protocol=dubbo + dubbo.service.com.ikurento.user.UserProvider.interface=com.ikurento.user.UserProvider + dubbo.service.com.ikurento.user.UserProvider.loadbalance=random + dubbo.service.com.ikurento.user.UserProvider.warmup=100 + dubbo.service.com.ikurento.user.UserProvider.cluster=failover +` + type mockDataListener struct { eventList []remoting.Event client *Client @@ -36,3 +66,41 @@ func (m *mockDataListener) DataChange(eventType remoting.Event) bool { } return true } + +func (s *KubernetesClientTestSuite) TestListener() { + + t := s.T() + + var tests = []struct { + input struct { + k string + v string + } + }{ + {input: struct { + k string + v string + }{k: "/dubbo", v: changedData}}, + } + + c := s.initClient() + defer c.Close() + + listener := NewEventListener(c) + dataListener := &mockDataListener{client: c, changedData: changedData, rc: make(chan remoting.Event)} + listener.ListenServiceEvent("/dubbo", dataListener) + + // NOTICE: direct listen will lose create msg + time.Sleep(time.Second) + for _, tc := range tests { + + k := tc.input.k + v := tc.input.v + if err := c.Create(k, v); err != nil { + t.Fatal(err) + } + + } + msg := <-dataListener.rc + assert.Equal(t, changedData, msg.Content) +} From dfa8267fda1f75ff96faa0caa360ff021b9c077c Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 15:01:57 +0800 Subject: [PATCH 26/61] sync watch unit goroutine --- remoting/etcdv3/coverage.html | 970 ----------------------------- remoting/kubernetes/client_test.go | 15 +- 2 files changed, 9 insertions(+), 976 deletions(-) delete mode 100644 remoting/etcdv3/coverage.html diff --git a/remoting/etcdv3/coverage.html b/remoting/etcdv3/coverage.html deleted file mode 100644 index 04a7ab97ca..0000000000 --- a/remoting/etcdv3/coverage.html +++ /dev/null @@ -1,970 +0,0 @@ - - - - - - - - -
- -
- not tracked - - not covered - covered - -
-
-
- - - - - - - -
- - - diff --git a/remoting/kubernetes/client_test.go b/remoting/kubernetes/client_test.go index 7345ddfbd5..aedd7ced83 100644 --- a/remoting/kubernetes/client_test.go +++ b/remoting/kubernetes/client_test.go @@ -395,19 +395,22 @@ func (s *KubernetesClientTestSuite) TestClientWatchPrefix() { go func() { - defer wg.Done() - wc, err := client.WatchWithPrefix(prefix) if err != nil { t.Fatal(err) } + wg.Done() + for e := range wc { t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) } }() + // must wait the watch goroutine work + wg.Wait() + for _, tc := range tests { k := tc.input.k @@ -419,7 +422,6 @@ func (s *KubernetesClientTestSuite) TestClientWatchPrefix() { } client.Close() - wg.Wait() } func (s *KubernetesClientTestSuite) TestNewClient() { @@ -444,12 +446,11 @@ func (s *KubernetesClientTestSuite) TestClientWatch() { go func() { - defer wg.Done() - wc, err := client.Watch(prefix) if err != nil { t.Fatal(err) } + wg.Done() for e := range wc { t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) @@ -457,6 +458,9 @@ func (s *KubernetesClientTestSuite) TestClientWatch() { }() + // must wait the watch goroutine already start the watch goroutine + wg.Wait() + for _, tc := range tests { k := tc.input.k @@ -468,7 +472,6 @@ func (s *KubernetesClientTestSuite) TestClientWatch() { } client.Close() - wg.Wait() } func TestKubernetesClient(t *testing.T) { From c7474fabc45d14a74983afcabebf4bf392c889bc Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 15:22:41 +0800 Subject: [PATCH 27/61] Fix method name bug,and handle the del event in config-listener. --- registry/etcdv3/listener.go | 2 +- registry/kubernetes/listener.go | 2 +- remoting/kubernetes/client.go | 10 ++++++++-- remoting/kubernetes/store.go | 16 ++++++++++------ 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/registry/etcdv3/listener.go b/registry/etcdv3/listener.go index f9b046a2c5..6fa84126f8 100644 --- a/registry/etcdv3/listener.go +++ b/registry/etcdv3/listener.go @@ -97,7 +97,7 @@ func (l *configurationListener) Next() (*registry.ServiceEvent, error) { case e := <-l.events: logger.Infof("got etcd event %#v", e) - if e.ConfigType == remoting.EventTypeDel { + if e.ConfigType == remoting.EventTypeDel && l.registry.client.Valid() { select { case <-l.registry.Done(): logger.Warnf("update @result{%s}. But its connection to registry is invalid", e.Value) diff --git a/registry/kubernetes/listener.go b/registry/kubernetes/listener.go index 6ddcd4f07d..c1d4f70fed 100644 --- a/registry/kubernetes/listener.go +++ b/registry/kubernetes/listener.go @@ -97,7 +97,7 @@ func (l *configurationListener) Next() (*registry.ServiceEvent, error) { case e := <-l.events: logger.Infof("got kubernetes event %#v", e) - if e.ConfigType == remoting.EventTypeDel { + if e.ConfigType == remoting.EventTypeDel && !l.registry.client.Valid() { select { case <-l.registry.Done(): logger.Warnf("update @result{%s}. But its connection to registry is invalid", e.Value) diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index 2091cc2f1e..90e74aa8f2 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -74,7 +74,7 @@ type Client struct { store Store // protect the wg && currentPod - lock sync.Mutex + lock sync.RWMutex // current pod status currentPod *v1.Pod // protect the maintenanceStatus loop && watcher @@ -636,8 +636,14 @@ func (c *Client) Valid() bool { case <-c.Done(): return false default: - return true } + c.lock.RLock() + if c.rawClient == nil { + c.lock.RUnlock() + return false + } + c.lock.RUnlock() + return true } // Done diff --git a/remoting/kubernetes/store.go b/remoting/kubernetes/store.go index 06f32cc0cb..51365c14c7 100644 --- a/remoting/kubernetes/store.go +++ b/remoting/kubernetes/store.go @@ -22,9 +22,7 @@ import ( "strconv" "strings" "sync" -) -import ( perrors "github.com/pkg/errors" ) @@ -112,14 +110,19 @@ type storeImpl struct { watchers map[uint64]*watcher } -func (s *storeImpl) loop() { +// on stop +// when the store was closed +func (s *storeImpl) onStop() { select { case <-s.ctx.Done(): + // parent ctx be canceled, close the store s.lock.Lock() - defer s.lock.Unlock() - for _, w := range s.watchers { + watchers := s.watchers + s.lock.Unlock() + + for _, w := range watchers { // stop data stream close(w.ch) // stop watcher @@ -145,6 +148,7 @@ func (s *storeImpl) Done() <-chan struct{} { func (s *storeImpl) Put(object *Object) error { sendMsg := func(object *Object, w *watcher) { + s.lock.Lock() defer s.lock.Unlock() select { @@ -325,6 +329,6 @@ func newStore(ctx context.Context) Store { cache: map[string]*Object{}, watchers: map[uint64]*watcher{}, } - go s.loop() + go s.onStop() return s } From d776f8e32629b0735876a6a974ccafbd533659ec Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 15:56:26 +0800 Subject: [PATCH 28/61] Fix remote/kubernetes sendMsg locker --- remoting/kubernetes/client.go | 12 +++---- remoting/kubernetes/client_test.go | 50 +++++++++++++++++++++--------- remoting/kubernetes/listener.go | 16 ++++++++-- remoting/kubernetes/store.go | 10 +++--- remoting/kubernetes/store_test.go | 23 +++++++++++--- 5 files changed, 78 insertions(+), 33 deletions(-) diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index 90e74aa8f2..34601e5601 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -605,26 +605,26 @@ func (c *Client) GetChildren(k string) ([]string, []string, error) { // Watch // watch on spec key -func (c *Client) Watch(k string) (<-chan *Object, error) { +func (c *Client) Watch(k string) (<-chan *Object, <-chan struct{}, error) { w, err := c.store.Watch(k, false) if err != nil { - return nil, perrors.WithMessagef(err, "watch on (%s)", k) + return nil, nil, perrors.WithMessagef(err, "watch on (%s)", k) } - return w.ResultChan(), nil + return w.ResultChan(), w.done(), nil } // Watch // watch on spec prefix -func (c *Client) WatchWithPrefix(prefix string) (<-chan *Object, error) { +func (c *Client) WatchWithPrefix(prefix string) (<-chan *Object, <-chan struct{}, error) { w, err := c.store.Watch(prefix, true) if err != nil { - return nil, perrors.WithMessagef(err, "watch on prefix (%s)", prefix) + return nil, nil, perrors.WithMessagef(err, "watch on prefix (%s)", prefix) } - return w.ResultChan(), nil + return w.ResultChan(), w.done(), nil } // Valid diff --git a/remoting/kubernetes/client_test.go b/remoting/kubernetes/client_test.go index aedd7ced83..49fcad608a 100644 --- a/remoting/kubernetes/client_test.go +++ b/remoting/kubernetes/client_test.go @@ -19,14 +19,16 @@ package kubernetes import ( "encoding/json" + "net/http" "os" "strings" "sync" "testing" "time" +) +import ( "github.com/stretchr/testify/suite" - v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" @@ -234,6 +236,8 @@ func (s *KubernetesClientTestSuite) SetupSuite() { if err := os.Setenv(nameSpaceKey, s.currentPod.GetNamespace()); err != nil { t.Fatal(err) } + + go http.ListenAndServe(":6061", nil) } func (s *KubernetesClientTestSuite) TestReadCurrentPodName() { @@ -329,16 +333,23 @@ func (s *KubernetesClientTestSuite) TestClientGetChildrenKVList() { go func() { defer wg.Done() - wc, err := client.WatchWithPrefix(prefix) + wc, done, err := client.WatchWithPrefix(prefix) if err != nil { t.Fatal(err) } i := 0 - for e := range wc { - i++ - t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) - if i == 3 { - // already sync all event + + for { + select { + case e := <-wc: + i++ + t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) + if i == 3 { + // already sync all event + return + } + case <-done: + t.Log("the store watcher was stopped") return } } @@ -395,17 +406,22 @@ func (s *KubernetesClientTestSuite) TestClientWatchPrefix() { go func() { - wc, err := client.WatchWithPrefix(prefix) + wc, done, err := client.WatchWithPrefix(prefix) if err != nil { t.Fatal(err) } wg.Done() - for e := range wc { - t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) + for { + select { + case e := <-wc: + t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) + case <-done: + t.Log("the store watcher was stopped") + return + } } - }() // must wait the watch goroutine work @@ -446,14 +462,20 @@ func (s *KubernetesClientTestSuite) TestClientWatch() { go func() { - wc, err := client.Watch(prefix) + wc, done, err := client.Watch(prefix) if err != nil { t.Fatal(err) } wg.Done() - for e := range wc { - t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) + for { + select { + case e := <-wc: + t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) + case <-done: + t.Log("the store watcher was stopped") + return + } } }() diff --git a/remoting/kubernetes/listener.go b/remoting/kubernetes/listener.go index da6d835159..0425aaf92d 100644 --- a/remoting/kubernetes/listener.go +++ b/remoting/kubernetes/listener.go @@ -52,7 +52,7 @@ func (l *EventListener) ListenServiceNodeEvent(key string, listener ...remoting. l.wg.Add(1) defer l.wg.Done() for { - wc, err := l.client.Watch(key) + wc, done, err := l.client.Watch(key) if err != nil { logger.Warnf("watch exist{key:%s} = error{%v}", key, err) return false @@ -65,7 +65,12 @@ func (l *EventListener) ListenServiceNodeEvent(key string, listener ...remoting. logger.Warnf("kubernetes client stopped") return false - // handle kubernetes-store events + // store watcher stopped + case <-done: + logger.Warnf("kubernetes store watcher stopped") + return false + + // handle kubernetes-store events case e, ok := <-wc: if !ok { logger.Warnf("kubernetes-store watch-chan closed") @@ -123,7 +128,7 @@ func (l *EventListener) ListenServiceNodeEventWithPrefix(prefix string, listener l.wg.Add(1) defer l.wg.Done() for { - wc, err := l.client.WatchWithPrefix(prefix) + wc, done, err := l.client.WatchWithPrefix(prefix) if err != nil { logger.Warnf("listenDirEvent(key{%s}) = error{%v}", prefix, err) } @@ -134,6 +139,11 @@ func (l *EventListener) ListenServiceNodeEventWithPrefix(prefix string, listener logger.Warnf("kubernetes client stopped") return + // watcher stopped + case <-done: + logger.Warnf("kubernetes store watcher stopped") + return + // kuberentes-store event stream case e, ok := <-wc: diff --git a/remoting/kubernetes/store.go b/remoting/kubernetes/store.go index 51365c14c7..a2cc994198 100644 --- a/remoting/kubernetes/store.go +++ b/remoting/kubernetes/store.go @@ -110,9 +110,9 @@ type storeImpl struct { watchers map[uint64]*watcher } -// on stop +// wait exit // when the store was closed -func (s *storeImpl) onStop() { +func (s *storeImpl) waitExit() { select { case <-s.ctx.Done(): @@ -124,7 +124,7 @@ func (s *storeImpl) onStop() { for _, w := range watchers { // stop data stream - close(w.ch) + // close(w.ch) // stop watcher w.stop() } @@ -149,8 +149,6 @@ func (s *storeImpl) Put(object *Object) error { sendMsg := func(object *Object, w *watcher) { - s.lock.Lock() - defer s.lock.Unlock() select { case <-w.done(): // the watcher already stop @@ -329,6 +327,6 @@ func newStore(ctx context.Context) Store { cache: map[string]*Object{}, watchers: map[uint64]*watcher{}, } - go s.onStop() + go s.waitExit() return s } diff --git a/remoting/kubernetes/store_test.go b/remoting/kubernetes/store_test.go index 11036adb40..fc6df0602c 100644 --- a/remoting/kubernetes/store_test.go +++ b/remoting/kubernetes/store_test.go @@ -44,8 +44,15 @@ func TestStore(t *testing.T) { if err != nil { t.Fatal(err) } - for e := range w.ResultChan() { - t.Logf("consumer %s got %s\n", w.ID(), e.Key) + for { + select { + case e := <-w.ResultChan(): + t.Logf("consumer %s got %s\n", w.ID(), e.Key) + + case <-w.done(): + t.Logf("consumer %s stopped", w.ID()) + return + } } }() } @@ -59,8 +66,16 @@ func TestStore(t *testing.T) { if err != nil { t.Fatal(err) } - for e := range w.ResultChan() { - t.Logf("prefix consumer %s got %s\n", w.ID(), e.Key) + + for { + select { + case e := <-w.ResultChan(): + t.Logf("prefix consumer %s got %s\n", w.ID(), e.Key) + + case <-w.done(): + t.Logf("prefix consumer %s stopped", w.ID()) + return + } } }() } From 088625775848fd99a9bf62a8779ac8008b3d3ef8 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 15:58:19 +0800 Subject: [PATCH 29/61] delete unused http pprof suite --- remoting/kubernetes/client_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/remoting/kubernetes/client_test.go b/remoting/kubernetes/client_test.go index 49fcad608a..0ffdda42a5 100644 --- a/remoting/kubernetes/client_test.go +++ b/remoting/kubernetes/client_test.go @@ -19,7 +19,6 @@ package kubernetes import ( "encoding/json" - "net/http" "os" "strings" "sync" @@ -237,7 +236,6 @@ func (s *KubernetesClientTestSuite) SetupSuite() { t.Fatal(err) } - go http.ListenAndServe(":6061", nil) } func (s *KubernetesClientTestSuite) TestReadCurrentPodName() { From bf8bf85dc37399a23d124a7e6809c5c76ee90d49 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 17:13:36 +0800 Subject: [PATCH 30/61] Rename watcher and store name --- remoting/kubernetes/client.go | 20 ++--- remoting/kubernetes/listener.go | 2 +- remoting/kubernetes/{store.go => watch.go} | 86 ++++++++++--------- .../{store_test.go => watch_test.go} | 4 +- 4 files changed, 57 insertions(+), 55 deletions(-) rename remoting/kubernetes/{store.go => watch.go} (79%) rename remoting/kubernetes/{store_test.go => watch_test.go} (97%) diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index 34601e5601..3e4b26c0ef 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -71,7 +71,7 @@ type Client struct { ns string // the memory store - store Store + store WatcherSet // protect the wg && currentPod lock sync.RWMutex @@ -132,7 +132,7 @@ func newMockClient(namespace string, mockClientGenerator func() (kubernetes.Inte ns: namespace, rawClient: rawClient, ctx: ctx, - store: newStore(ctx), + store: newWatcherSet(ctx), cancel: cancel, } @@ -185,7 +185,7 @@ func newClient(namespace string) (*Client, error) { cfg: cfg, rawClient: rawClient, ctx: ctx, - store: newStore(ctx), + store: newWatcherSet(ctx), cancel: cancel, } @@ -415,10 +415,10 @@ func (c *Client) handleWatchedPodEvent(p *v1.Pod, eventType watch.EventType) { // unmarshalRecord // unmarshal the kubernetes dubbo annotation value -func (c *Client) unmarshalRecord(record string) ([]*Object, error) { +func (c *Client) unmarshalRecord(record string) ([]*WatcherEvent, error) { if len(record) == 0 { - // []*Object is nil. + // []*WatcherEvent is nil. return nil, nil } @@ -427,7 +427,7 @@ func (c *Client) unmarshalRecord(record string) ([]*Object, error) { return nil, perrors.WithMessagef(err, "decode record (%s)", record) } - var out []*Object + var out []*WatcherEvent if err := json.Unmarshal(rawMsg, &out); err != nil { return nil, perrors.WithMessage(err, "decode json") } @@ -436,7 +436,7 @@ func (c *Client) unmarshalRecord(record string) ([]*Object, error) { // marshalRecord // marshal the kubernetes dubbo annotation value -func (c *Client) marshalRecord(ol []*Object) (string, error) { +func (c *Client) marshalRecord(ol []*WatcherEvent) (string, error) { msg, err := json.Marshal(ol) if err != nil { @@ -552,7 +552,7 @@ func (c *Client) assembleDUBBOAnnotations(k, v string, currentPod *v1.Pod) (oldP return } - newAnnotations, err := c.marshalRecord(append(al, &Object{Key: k, Value: v})) + newAnnotations, err := c.marshalRecord(append(al, &WatcherEvent{Key: k, Value: v})) if err != nil { err = perrors.WithMessage(err, "marshal record") return @@ -605,7 +605,7 @@ func (c *Client) GetChildren(k string) ([]string, []string, error) { // Watch // watch on spec key -func (c *Client) Watch(k string) (<-chan *Object, <-chan struct{}, error) { +func (c *Client) Watch(k string) (<-chan *WatcherEvent, <-chan struct{}, error) { w, err := c.store.Watch(k, false) if err != nil { @@ -617,7 +617,7 @@ func (c *Client) Watch(k string) (<-chan *Object, <-chan struct{}, error) { // Watch // watch on spec prefix -func (c *Client) WatchWithPrefix(prefix string) (<-chan *Object, <-chan struct{}, error) { +func (c *Client) WatchWithPrefix(prefix string) (<-chan *WatcherEvent, <-chan struct{}, error) { w, err := c.store.Watch(prefix, true) if err != nil { diff --git a/remoting/kubernetes/listener.go b/remoting/kubernetes/listener.go index 0425aaf92d..83b84785e8 100644 --- a/remoting/kubernetes/listener.go +++ b/remoting/kubernetes/listener.go @@ -89,7 +89,7 @@ func (l *EventListener) ListenServiceNodeEvent(key string, listener ...remoting. // return true mean the event type is DELETE // return false mean the event type is CREATE || UPDATE -func (l *EventListener) handleEvents(event *Object, listeners ...remoting.DataListener) bool { +func (l *EventListener) handleEvents(event *WatcherEvent, listeners ...remoting.DataListener) bool { logger.Infof("got a kubernetes-store event {type: %d, key: %s}", event.EventType, event.Key) diff --git a/remoting/kubernetes/store.go b/remoting/kubernetes/watch.go similarity index 79% rename from remoting/kubernetes/store.go rename to remoting/kubernetes/watch.go index a2cc994198..4f36b3522c 100644 --- a/remoting/kubernetes/store.go +++ b/remoting/kubernetes/watch.go @@ -22,7 +22,9 @@ import ( "strconv" "strings" "sync" +) +import ( perrors "github.com/pkg/errors" ) @@ -57,9 +59,9 @@ func (e eventType) String() string { } } -// Object +// WatcherEvent // object is element in store -type Object struct { +type WatcherEvent struct { // event-type EventType eventType `json:"-"` // the dubbo-go should consume the key @@ -68,14 +70,14 @@ type Object struct { Value string `json:"v"` } -// Watchable Store -type Store interface { +// Watchable WatcherSet +type WatcherSet interface { // put the object to the store - Put(object *Object) error + Put(object *WatcherEvent) error // if prefix is false, - // the len([]*Object) == 1 - Get(key string, prefix bool) ([]*Object, error) + // the len([]*WatcherEvent) == 1 + Get(key string, prefix bool) ([]*WatcherEvent, error) // watch the spec key or key prefix Watch(key string, prefix bool) (Watcher, error) // check the store status @@ -87,7 +89,7 @@ type Watcher interface { // the watcher's id ID() string // result stream - ResultChan() <-chan *Object + ResultChan() <-chan *WatcherEvent // Stop the watcher stop() // check the watcher status @@ -95,7 +97,7 @@ type Watcher interface { } // the store -type storeImpl struct { +type watcherSetImpl struct { // Client's ctx, client die, the store will die too ctx context.Context @@ -104,15 +106,15 @@ type storeImpl struct { lock sync.RWMutex // the key is dubbo-go interest meta - cache map[string]*Object + cache map[string]*WatcherEvent currentWatcherId uint64 watchers map[uint64]*watcher } -// wait exit +// closeWatchers // when the store was closed -func (s *storeImpl) waitExit() { +func (s *watcherSetImpl) closeWatchers() { select { case <-s.ctx.Done(): @@ -133,21 +135,21 @@ func (s *storeImpl) waitExit() { // Watch // watch on spec key, with or without prefix -func (s *storeImpl) Watch(key string, prefix bool) (Watcher, error) { +func (s *watcherSetImpl) Watch(key string, prefix bool) (Watcher, error) { return s.addWatcher(key, prefix) } // Done // get the store status -func (s *storeImpl) Done() <-chan struct{} { +func (s *watcherSetImpl) Done() <-chan struct{} { return s.ctx.Done() } // Put // put the object to store -func (s *storeImpl) Put(object *Object) error { +func (s *watcherSetImpl) Put(object *WatcherEvent) error { - sendMsg := func(object *Object, w *watcher) { + sendMsg := func(object *WatcherEvent, w *watcher) { select { case <-w.done(): @@ -202,8 +204,7 @@ func (s *storeImpl) Put(object *Object) error { } // valid -// valid the client status should protected by lock -func (s *storeImpl) valid() error { +func (s *watcherSetImpl) valid() error { select { case <-s.ctx.Done(): return ErrStoreAlreadyStopped @@ -213,34 +214,35 @@ func (s *storeImpl) valid() error { } // addWatcher -func (s *storeImpl) addWatcher(key string, prefix bool) (Watcher, error) { +func (s *watcherSetImpl) addWatcher(key string, prefix bool) (Watcher, error) { + + if err := s.valid(); err != nil { + return nil, err + } + + s.lock.Lock() + defer s.lock.Unlock() + + // increase the watcher-id + s.currentWatcherId++ w := &watcher{ + id: s.currentWatcherId, store: s, interested: struct { key string prefix bool }{key: key, prefix: prefix}, - ch: make(chan *Object, defaultWatcherChanSize), + ch: make(chan *WatcherEvent, defaultWatcherChanSize), exit: make(chan struct{}), } - - s.lock.Lock() - defer s.lock.Unlock() - - if err := s.valid(); err != nil { - return nil, err - } - s.watchers[s.currentWatcherId] = w - w.id = s.currentWatcherId - s.currentWatcherId = s.currentWatcherId + 1 return w, nil } // Get // get elements from cache -func (s *storeImpl) Get(key string, prefix bool) ([]*Object, error) { +func (s *watcherSetImpl) Get(key string, prefix bool) ([]*WatcherEvent, error) { s.lock.RLock() defer s.lock.RUnlock() @@ -252,14 +254,14 @@ func (s *storeImpl) Get(key string, prefix bool) ([]*Object, error) { if !prefix { for k, v := range s.cache { if k == key { - return []*Object{v}, nil + return []*WatcherEvent{v}, nil } } // object return nil, ErrKVPairNotFound } - var out []*Object + var out []*WatcherEvent for k, v := range s.cache { if strings.Contains(k, key) { @@ -279,21 +281,21 @@ type watcher struct { id uint64 // the underlay store - store *storeImpl + store *watcherSetImpl // the interest topic interested struct { key string prefix bool } - ch chan *Object + ch chan *WatcherEvent closeOnce sync.Once exit chan struct{} } // ResultChan -func (w *watcher) ResultChan() <-chan *Object { +func (w *watcher) ResultChan() <-chan *WatcherEvent { return w.ch } @@ -319,14 +321,14 @@ func (w *watcher) done() <-chan struct{} { return w.exit } -// newStore -// new store from parent context -func newStore(ctx context.Context) Store { - s := &storeImpl{ +// newWatcherSet +// new watcher set from parent context +func newWatcherSet(ctx context.Context) WatcherSet { + s := &watcherSetImpl{ ctx: ctx, - cache: map[string]*Object{}, + cache: map[string]*WatcherEvent{}, watchers: map[uint64]*watcher{}, } - go s.waitExit() + go s.closeWatchers() return s } diff --git a/remoting/kubernetes/store_test.go b/remoting/kubernetes/watch_test.go similarity index 97% rename from remoting/kubernetes/store_test.go rename to remoting/kubernetes/watch_test.go index fc6df0602c..10fbc7eb53 100644 --- a/remoting/kubernetes/store_test.go +++ b/remoting/kubernetes/watch_test.go @@ -30,7 +30,7 @@ func TestStore(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() - s := newStore(ctx) + s := newWatcherSet(ctx) wg := sync.WaitGroup{} @@ -82,7 +82,7 @@ func TestStore(t *testing.T) { for i := 0; i < 5; i++ { go func(i int) { - if err := s.Put(&Object{ + if err := s.Put(&WatcherEvent{ Key: "key-" + strconv.Itoa(i), Value: strconv.Itoa(i), }); err != nil { From bc536bb194e6adc3fa450e870fd5c13da0e80328 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 17:59:26 +0800 Subject: [PATCH 31/61] Fix named err --- remoting/kubernetes/client.go | 64 +++++++++++++++--------------- remoting/kubernetes/client_test.go | 6 +-- remoting/kubernetes/listener.go | 26 ++++++------ remoting/kubernetes/watch.go | 62 ++++++++++++++--------------- 4 files changed, 79 insertions(+), 79 deletions(-) diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index 3e4b26c0ef..26bf627d4b 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -70,14 +70,14 @@ type Client struct { ns string - // the memory store - store WatcherSet + // the memory watcherSet + watcherSet WatcherSet // protect the wg && currentPod lock sync.RWMutex // current pod status currentPod *v1.Pod - // protect the maintenanceStatus loop && watcher + // protect the watchPods loop && watcher wg sync.WaitGroup // manage the client lifecycle @@ -132,7 +132,7 @@ func newMockClient(namespace string, mockClientGenerator func() (kubernetes.Inte ns: namespace, rawClient: rawClient, ctx: ctx, - store: newWatcherSet(ctx), + watcherSet: newWatcherSet(ctx), cancel: cancel, } @@ -144,13 +144,13 @@ func newMockClient(namespace string, mockClientGenerator func() (kubernetes.Inte // record current status c.currentPod = currentPod - // init the store by current pods - if err := c.initStore(); err != nil { - return nil, perrors.WithMessage(err, "init store") + // init the watcherSet by current pods + if err := c.initWatchSet(); err != nil { + return nil, perrors.WithMessage(err, "init watcherSet") } // start kubernetes watch loop - if err := c.maintenanceStatus(); err != nil { + if err := c.watchPods(); err != nil { return nil, perrors.WithMessage(err, "maintenance the kubernetes status") } @@ -185,7 +185,7 @@ func newClient(namespace string) (*Client, error) { cfg: cfg, rawClient: rawClient, ctx: ctx, - store: newWatcherSet(ctx), + watcherSet: newWatcherSet(ctx), cancel: cancel, } @@ -197,13 +197,13 @@ func newClient(namespace string) (*Client, error) { // record current status c.currentPod = currentPod - // init the store by current pods - if err := c.initStore(); err != nil { - return nil, perrors.WithMessage(err, "init store") + // init the watcherSet by current pods + if err := c.initWatchSet(); err != nil { + return nil, perrors.WithMessage(err, "init watcherSet") } // start kubernetes watch loop - if err := c.maintenanceStatus(); err != nil { + if err := c.watchPods(); err != nil { return nil, perrors.WithMessage(err, "maintenance the kubernetes status") } @@ -243,10 +243,10 @@ func (c *Client) initCurrentPod() (*v1.Pod, error) { return currentPod, nil } -// initStore +// initWatchSet // 1. get all with dubbo label pods -// 2. put every element to store -func (c *Client) initStore() error { +// 2. put every element to watcherSet +func (c *Client) initWatchSet() error { pods, err := c.rawClient.CoreV1().Pods(c.ns).List(metav1.ListOptions{ LabelSelector: fields.OneTermEqualSelector(DubboIOLabelKey, DubboIOLabelValue).String(), @@ -263,9 +263,9 @@ func (c *Client) initStore() error { return nil } -// maintenanceStatus +// watchPods // try to watch kubernetes pods -func (c *Client) maintenanceStatus() error { +func (c *Client) watchPods() error { // try once watcher, err := c.rawClient.CoreV1().Pods(c.ns).Watch(metav1.ListOptions{ @@ -280,18 +280,18 @@ func (c *Client) maintenanceStatus() error { c.wg.Add(1) // add wg, grace close the client - go c.maintenanceStatusLoop() + go c.watchPodsLoop() return nil } -// maintenanceStatus +// watchPods // try to notify -func (c *Client) maintenanceStatusLoop() { +func (c *Client) watchPodsLoop() { defer func() { // notify other goroutine, this loop over c.wg.Done() - logger.Info("maintenanceStatusLoop goroutine game over") + logger.Info("watchPodsLoop goroutine game over") }() var lastResourceVersion string @@ -401,10 +401,10 @@ func (c *Client) handleWatchedPodEvent(p *v1.Pod, eventType watch.EventType) { return } - logger.Debugf("prepare to put object (%#v) to kuberentes-store", o) + logger.Debugf("prepare to put object (%#v) to kuberentes-watcherSet", o) - if err := c.store.Put(o); err != nil { - logger.Errorf("put (%#v) to cache store: %v ", o, err) + if err := c.watcherSet.Put(o); err != nil { + logger.Errorf("put (%#v) to cache watcherSet: %v ", o, err) return } @@ -487,7 +487,7 @@ func (c *Client) Create(k, v string) error { } c.currentPod = updatedPod - // not update the store, the store should be write by the maintenanceStatusLoop + // not update the watcherSet, the watcherSet should be write by the watchPodsLoop return nil } @@ -584,12 +584,12 @@ func (c *Client) getPatch(oldPod, newPod *v1.Pod) ([]byte, error) { } // GetChildren -// get k children list from kubernetes-store +// get k children list from kubernetes-watcherSet func (c *Client) GetChildren(k string) ([]string, []string, error) { - objectList, err := c.store.Get(k, true) + objectList, err := c.watcherSet.Get(k, true) if err != nil { - return nil, nil, perrors.WithMessagef(err, "get children from store on (%s)", k) + return nil, nil, perrors.WithMessagef(err, "get children from watcherSet on (%s)", k) } var kList []string @@ -607,7 +607,7 @@ func (c *Client) GetChildren(k string) ([]string, []string, error) { // watch on spec key func (c *Client) Watch(k string) (<-chan *WatcherEvent, <-chan struct{}, error) { - w, err := c.store.Watch(k, false) + w, err := c.watcherSet.Watch(k, false) if err != nil { return nil, nil, perrors.WithMessagef(err, "watch on (%s)", k) } @@ -619,7 +619,7 @@ func (c *Client) Watch(k string) (<-chan *WatcherEvent, <-chan struct{}, error) // watch on spec prefix func (c *Client) WatchWithPrefix(prefix string) (<-chan *WatcherEvent, <-chan struct{}, error) { - w, err := c.store.Watch(prefix, true) + w, err := c.watcherSet.Watch(prefix, true) if err != nil { return nil, nil, perrors.WithMessagef(err, "watch on prefix (%s)", prefix) } @@ -665,7 +665,7 @@ func (c *Client) Close() { c.cancel() // the client ctx be canceled - // will trigger the store watchers all stopped + // will trigger the watcherSet watchers all stopped // so, just wait c.wg.Wait() } diff --git a/remoting/kubernetes/client_test.go b/remoting/kubernetes/client_test.go index 0ffdda42a5..d8ed782919 100644 --- a/remoting/kubernetes/client_test.go +++ b/remoting/kubernetes/client_test.go @@ -347,7 +347,7 @@ func (s *KubernetesClientTestSuite) TestClientGetChildrenKVList() { return } case <-done: - t.Log("the store watcher was stopped") + t.Log("the watcherSet watcher was stopped") return } } @@ -416,7 +416,7 @@ func (s *KubernetesClientTestSuite) TestClientWatchPrefix() { case e := <-wc: t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) case <-done: - t.Log("the store watcher was stopped") + t.Log("the watcherSet watcher was stopped") return } } @@ -471,7 +471,7 @@ func (s *KubernetesClientTestSuite) TestClientWatch() { case e := <-wc: t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) case <-done: - t.Log("the store watcher was stopped") + t.Log("the watcherSet watcher was stopped") return } } diff --git a/remoting/kubernetes/listener.go b/remoting/kubernetes/listener.go index 83b84785e8..575734ab30 100644 --- a/remoting/kubernetes/listener.go +++ b/remoting/kubernetes/listener.go @@ -65,15 +65,15 @@ func (l *EventListener) ListenServiceNodeEvent(key string, listener ...remoting. logger.Warnf("kubernetes client stopped") return false - // store watcher stopped + // watcherSet watcher stopped case <-done: - logger.Warnf("kubernetes store watcher stopped") + logger.Warnf("kubernetes watcherSet watcher stopped") return false - // handle kubernetes-store events + // handle kubernetes-watcherSet events case e, ok := <-wc: if !ok { - logger.Warnf("kubernetes-store watch-chan closed") + logger.Warnf("kubernetes-watcherSet watch-chan closed") return false } @@ -91,12 +91,12 @@ func (l *EventListener) ListenServiceNodeEvent(key string, listener ...remoting. // return false mean the event type is CREATE || UPDATE func (l *EventListener) handleEvents(event *WatcherEvent, listeners ...remoting.DataListener) bool { - logger.Infof("got a kubernetes-store event {type: %d, key: %s}", event.EventType, event.Key) + logger.Infof("got a kubernetes-watcherSet event {type: %d, key: %s}", event.EventType, event.Key) switch event.EventType { case Create: for _, listener := range listeners { - logger.Infof("kubernetes-store get event (key{%s}) = event{EventNodeDataCreated}", event.Key) + logger.Infof("kubernetes-watcherSet get event (key{%s}) = event{EventNodeDataCreated}", event.Key) listener.DataChange(remoting.Event{ Path: string(event.Key), Action: remoting.EventTypeAdd, @@ -106,7 +106,7 @@ func (l *EventListener) handleEvents(event *WatcherEvent, listeners ...remoting. return false case Update: for _, listener := range listeners { - logger.Infof("kubernetes-store get event (key{%s}) = event{EventNodeDataChanged}", event.Key) + logger.Infof("kubernetes-watcherSet get event (key{%s}) = event{EventNodeDataChanged}", event.Key) listener.DataChange(remoting.Event{ Path: string(event.Key), Action: remoting.EventTypeUpdate, @@ -115,7 +115,7 @@ func (l *EventListener) handleEvents(event *WatcherEvent, listeners ...remoting. } return false case Delete: - logger.Warnf("kubernetes-store get event (key{%s}) = event{EventNodeDeleted}", event.Key) + logger.Warnf("kubernetes-watcherSet get event (key{%s}) = event{EventNodeDeleted}", event.Key) return true default: return false @@ -141,14 +141,14 @@ func (l *EventListener) ListenServiceNodeEventWithPrefix(prefix string, listener // watcher stopped case <-done: - logger.Warnf("kubernetes store watcher stopped") + logger.Warnf("kubernetes watcherSet watcher stopped") return - // kuberentes-store event stream + // kuberentes-watcherSet event stream case e, ok := <-wc: if !ok { - logger.Warnf("kubernetes-store watch-chan closed") + logger.Warnf("kubernetes-watcherSet watch-chan closed") return } @@ -171,7 +171,7 @@ func (l *EventListener) ListenServiceEvent(key string, listener remoting.DataLis _, ok := l.keyMap[key] l.keyMapLock.RUnlock() if ok { - logger.Warnf("kubernetes-store key %s has already been listened.", key) + logger.Warnf("kubernetes-watcherSet key %s has already been listened.", key) return } @@ -201,7 +201,7 @@ func (l *EventListener) ListenServiceEvent(key string, listener remoting.DataLis }) } - logger.Infof("listen dubbo provider key{%s} event and wait to get all provider from kubernetes-store", key) + logger.Infof("listen dubbo provider key{%s} event and wait to get all provider from kubernetes-watcherSet", key) go func(key string, listener remoting.DataListener) { l.ListenServiceNodeEventWithPrefix(key, listener) logger.Warnf("listenDirEvent(key{%s}) goroutine exit now", key) diff --git a/remoting/kubernetes/watch.go b/remoting/kubernetes/watch.go index 4f36b3522c..835bcdb9b7 100644 --- a/remoting/kubernetes/watch.go +++ b/remoting/kubernetes/watch.go @@ -29,8 +29,8 @@ import ( ) var ( - ErrStoreAlreadyStopped = perrors.New("the store already be stopped") - ErrKVPairNotFound = perrors.New("k/v pair not found") + ErrWatcherSetAlreadyStopped = perrors.New("the watcher-set already be stopped") + ErrKVPairNotFound = perrors.New("k/v pair not found") ) const ( @@ -60,7 +60,7 @@ func (e eventType) String() string { } // WatcherEvent -// object is element in store +// watch event is element in watcherSet type WatcherEvent struct { // event-type EventType eventType `json:"-"` @@ -73,14 +73,14 @@ type WatcherEvent struct { // Watchable WatcherSet type WatcherSet interface { - // put the object to the store + // put the watch event to the watch set Put(object *WatcherEvent) error // if prefix is false, // the len([]*WatcherEvent) == 1 Get(key string, prefix bool) ([]*WatcherEvent, error) // watch the spec key or key prefix Watch(key string, prefix bool) (Watcher, error) - // check the store status + // check the watcher set status Done() <-chan struct{} } @@ -96,13 +96,13 @@ type Watcher interface { done() <-chan struct{} } -// the store +// the watch set implement type watcherSetImpl struct { - // Client's ctx, client die, the store will die too + // Client's ctx, client die, the watch set will die too ctx context.Context - // protect store and watchers + // protect watcher-set and watchers lock sync.RWMutex // the key is dubbo-go interest meta @@ -113,13 +113,13 @@ type watcherSetImpl struct { } // closeWatchers -// when the store was closed +// when the watcher-set was closed func (s *watcherSetImpl) closeWatchers() { select { case <-s.ctx.Done(): - // parent ctx be canceled, close the store + // parent ctx be canceled, close the watch-set's watchers s.lock.Lock() watchers := s.watchers s.lock.Unlock() @@ -140,14 +140,14 @@ func (s *watcherSetImpl) Watch(key string, prefix bool) (Watcher, error) { } // Done -// get the store status +// get the watcher-set status func (s *watcherSetImpl) Done() <-chan struct{} { return s.ctx.Done() } // Put -// put the object to store -func (s *watcherSetImpl) Put(object *WatcherEvent) error { +// put the watch event to watcher-set +func (s *watcherSetImpl) Put(watcherEvent *WatcherEvent) error { sendMsg := func(object *WatcherEvent, w *watcher) { @@ -166,39 +166,39 @@ func (s *watcherSetImpl) Put(object *WatcherEvent) error { return err } - // put to store - if object.EventType == Delete { - delete(s.cache, object.Key) + // put to watcher-set + if watcherEvent.EventType == Delete { + delete(s.cache, watcherEvent.Key) } else { - old, ok := s.cache[object.Key] + old, ok := s.cache[watcherEvent.Key] if ok { - if old.Value == object.Value { + if old.Value == watcherEvent.Value { // already have this k/v pair return nil } } - // refresh the object - s.cache[object.Key] = object + // refresh the watcherEvent + s.cache[watcherEvent.Key] = watcherEvent } // notify watcher for _, w := range s.watchers { - if !strings.Contains(object.Key, w.interested.key) { + if !strings.Contains(watcherEvent.Key, w.interested.key) { // this watcher no interest in this element continue } if !w.interested.prefix { - if object.Key == w.interested.key { - go sendMsg(object, w) + if watcherEvent.Key == w.interested.key { + go sendMsg(watcherEvent, w) } // not interest continue } - go sendMsg(object, w) + go sendMsg(watcherEvent, w) } return nil } @@ -207,7 +207,7 @@ func (s *watcherSetImpl) Put(object *WatcherEvent) error { func (s *watcherSetImpl) valid() error { select { case <-s.ctx.Done(): - return ErrStoreAlreadyStopped + return ErrWatcherSetAlreadyStopped default: return nil } @@ -227,8 +227,8 @@ func (s *watcherSetImpl) addWatcher(key string, prefix bool) (Watcher, error) { s.currentWatcherId++ w := &watcher{ - id: s.currentWatcherId, - store: s, + id: s.currentWatcherId, + watcherSet: s, interested: struct { key string prefix bool @@ -241,7 +241,7 @@ func (s *watcherSetImpl) addWatcher(key string, prefix bool) (Watcher, error) { } // Get -// get elements from cache +// get elements from watcher-set func (s *watcherSetImpl) Get(key string, prefix bool) ([]*WatcherEvent, error) { s.lock.RLock() @@ -276,12 +276,12 @@ func (s *watcherSetImpl) Get(key string, prefix bool) ([]*WatcherEvent, error) { return out, nil } -// the store watcher +// the watcher-set watcher type watcher struct { id uint64 - // the underlay store - store *watcherSetImpl + // the underlay watcherSet + watcherSet *watcherSetImpl // the interest topic interested struct { From 7b13b44ca16a4b2b88bc5eee0ecd9ad9634714c1 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 18:00:02 +0800 Subject: [PATCH 32/61] Fix named err --- remoting/kubernetes/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index 26bf627d4b..ef77b4d11d 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -151,7 +151,7 @@ func newMockClient(namespace string, mockClientGenerator func() (kubernetes.Inte // start kubernetes watch loop if err := c.watchPods(); err != nil { - return nil, perrors.WithMessage(err, "maintenance the kubernetes status") + return nil, perrors.WithMessage(err, "watch pods") } logger.Info("init kubernetes registry success") @@ -204,7 +204,7 @@ func newClient(namespace string) (*Client, error) { // start kubernetes watch loop if err := c.watchPods(); err != nil { - return nil, perrors.WithMessage(err, "maintenance the kubernetes status") + return nil, perrors.WithMessage(err, "watch pods") } logger.Info("init kubernetes registry success") From b9bf7d984d5cdc9face2ad60b83bd34867d69b38 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 18:11:51 +0800 Subject: [PATCH 33/61] Fix time gap --- remoting/kubernetes/client.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index ef77b4d11d..9be25cb0ee 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -70,6 +70,9 @@ type Client struct { ns string + // current resource version + lastResourceVersion string + // the memory watcherSet watcherSet WatcherSet @@ -255,6 +258,9 @@ func (c *Client) initWatchSet() error { return perrors.WithMessagef(err, "list pods in namespace (%s)", c.ns) } + // set resource version + c.lastResourceVersion = pods.GetResourceVersion() + for _, pod := range pods.Items { logger.Debugf("got the pod (name: %s), (label: %v), (annotations: %v)", pod.Name, pod.GetLabels(), pod.GetAnnotations()) c.handleWatchedPodEvent(&pod, watch.Added) @@ -269,8 +275,9 @@ func (c *Client) watchPods() error { // try once watcher, err := c.rawClient.CoreV1().Pods(c.ns).Watch(metav1.ListOptions{ - LabelSelector: fields.OneTermEqualSelector(DubboIOLabelKey, DubboIOLabelValue).String(), - Watch: true, + LabelSelector: fields.OneTermEqualSelector(DubboIOLabelKey, DubboIOLabelValue).String(), + Watch: true, + ResourceVersion: c.lastResourceVersion, }) if err != nil { return perrors.WithMessagef(err, "try to watch the namespace (%s) pods", c.ns) @@ -294,14 +301,12 @@ func (c *Client) watchPodsLoop() { logger.Info("watchPodsLoop goroutine game over") }() - var lastResourceVersion string - for { wc, err := c.rawClient.CoreV1().Pods(c.ns).Watch(metav1.ListOptions{ LabelSelector: fields.OneTermEqualSelector(DubboIOLabelKey, DubboIOLabelValue).String(), Watch: true, - ResourceVersion: lastResourceVersion, + ResourceVersion: c.lastResourceVersion, }) if err != nil { logger.Warnf("watch the namespace (%s) pods: %v, retry after 2 seconds", c.ns, err) @@ -309,7 +314,7 @@ func (c *Client) watchPodsLoop() { continue } - logger.Infof("the old kubernetes client broken, collect the resource status from resource version (%s)", lastResourceVersion) + logger.Infof("the old kubernetes client broken, collect the resource status from resource version (%s)", c.lastResourceVersion) select { case <-c.ctx.Done(): @@ -350,8 +355,8 @@ func (c *Client) watchPodsLoop() { } // record the last resource version avoid to sync all pod - lastResourceVersion = o.GetResourceVersion() - logger.Infof("kuberentes get the current resource version %v", lastResourceVersion) + c.lastResourceVersion = o.GetResourceVersion() + logger.Infof("kubernetes get the current resource version %v", c.lastResourceVersion) // check event object type p, ok := event.Object.(*v1.Pod) From aee4f9056fd7262af72c1ac1b3e896eddcb7d8d9 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 18:25:39 +0800 Subject: [PATCH 34/61] delete unused select case --- remoting/kubernetes/client.go | 90 ++++++++++++++++------------------- 1 file changed, 41 insertions(+), 49 deletions(-) diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index 9be25cb0ee..dcb28c37d2 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -291,6 +291,10 @@ func (c *Client) watchPods() error { return nil } +type resourceVersionGetter interface { + GetResourceVersion() string +} + // watchPods // try to notify func (c *Client) watchPodsLoop() { @@ -316,61 +320,49 @@ func (c *Client) watchPodsLoop() { logger.Infof("the old kubernetes client broken, collect the resource status from resource version (%s)", c.lastResourceVersion) - select { - case <-c.ctx.Done(): - // the client stopped - logger.Info("the kubernetes client stopped") - return - - default: + for { + select { + // double check ctx + case <-c.ctx.Done(): + logger.Info("the kubernetes client stopped, resultChan len %d", len(wc.ResultChan())) + return - for { - select { - // double check ctx - case <-c.ctx.Done(): - logger.Info("the kubernetes client stopped") + // get one element from result-chan + case event, ok := <-wc.ResultChan(): + if !ok { + wc.Stop() + logger.Info("kubernetes watch chan die, create new") goto onceWatch + } - // get one element from result-chan - case event, ok := <-wc.ResultChan(): - if !ok { - wc.Stop() - logger.Info("kubernetes watch chan die, create new") - goto onceWatch - } - - if event.Type == watch.Error { - // watched a error event - logger.Warnf("kubernetes watch api report err (%#v)", event) - continue - } - - type resourceVersionGetter interface { - GetResourceVersion() string - } - - o, ok := event.Object.(resourceVersionGetter) - if !ok { - continue - } - - // record the last resource version avoid to sync all pod - c.lastResourceVersion = o.GetResourceVersion() - logger.Infof("kubernetes get the current resource version %v", c.lastResourceVersion) - - // check event object type - p, ok := event.Object.(*v1.Pod) - if !ok { - // not a pod - continue - } - - // handle the watched pod - go c.handleWatchedPodEvent(p, event.Type) + if event.Type == watch.Error { + // watched a error event + logger.Warnf("kubernetes watch api report err (%#v)", event) + continue } + + o, ok := event.Object.(resourceVersionGetter) + if !ok { + logger.Warnf("kubernetes response object not a versioned object") + continue + } + + // record the last resource version avoid to sync all pod + c.lastResourceVersion = o.GetResourceVersion() + logger.Infof("kubernetes get the current resource version %v", c.lastResourceVersion) + + // check event object type + p, ok := event.Object.(*v1.Pod) + if !ok { + // not a pod + continue + } + + // handle the watched pod + go c.handleWatchedPodEvent(p, event.Type) } - onceWatch: } + onceWatch: } } From 06b0da8589c439caa8d1b7be154e8bdcc9708163 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 18:28:34 +0800 Subject: [PATCH 35/61] Add more rich log --- remoting/kubernetes/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index dcb28c37d2..605a05fb5f 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -343,7 +343,7 @@ func (c *Client) watchPodsLoop() { o, ok := event.Object.(resourceVersionGetter) if !ok { - logger.Warnf("kubernetes response object not a versioned object") + logger.Warnf("kubernetes response object not a versioned object, its real type %T", event.Object) continue } @@ -354,7 +354,7 @@ func (c *Client) watchPodsLoop() { // check event object type p, ok := event.Object.(*v1.Pod) if !ok { - // not a pod + logger.Warnf("kubernetes response object not a Pod, its real type %T", event.Object) continue } From 83d397588c33fe167c06fb6c09b7670bb35c3600 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 20:51:43 +0800 Subject: [PATCH 36/61] move handle-client-restart from remote to registry --- registry/kubernetes/listener_test.go | 4 ++ registry/kubernetes/registry.go | 71 ++++++++++++++++++++++++---- registry/kubernetes/registry_test.go | 10 ++-- remoting/kubernetes/client.go | 31 +++--------- remoting/kubernetes/facade.go | 71 ---------------------------- remoting/kubernetes/facade_test.go | 40 ++-------------- remoting/kubernetes/listener.go | 5 -- 7 files changed, 84 insertions(+), 148 deletions(-) diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go index f89154e32e..88e1589020 100644 --- a/registry/kubernetes/listener_test.go +++ b/registry/kubernetes/listener_test.go @@ -19,6 +19,8 @@ package kubernetes import ( "encoding/json" + "net/http" + _ "net/http/pprof" "os" "strconv" "testing" @@ -238,6 +240,8 @@ func (s *KubernetesRegistryTestSuite) SetupSuite() { t.Fatal(err) } + go http.ListenAndServe(":6061", nil) + } func (s *KubernetesRegistryTestSuite) TestDataChange() { diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index 2a63660cc0..79f066a9bd 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -23,9 +23,11 @@ import ( "path" "strings" "sync" + "time" ) import ( + "github.com/dubbogo/getty" "github.com/dubbogo/gost/net" perrors "github.com/pkg/errors" k8s "k8s.io/client-go/kubernetes" @@ -46,7 +48,9 @@ var ( ) const ( - Name = "kubernetes" + Name = "kubernetes" + ConnDelay = 3 + MaxFailTimes = 15 ) func init() { @@ -57,7 +61,7 @@ func init() { type kubernetesRegistry struct { registry.BaseRegistry - cltLock sync.Mutex + cltLock sync.RWMutex client *kubernetes.Client listenerLock sync.Mutex listener *kubernetes.EventListener @@ -66,13 +70,15 @@ type kubernetesRegistry struct { } func (r *kubernetesRegistry) Client() *kubernetes.Client { - return r.client + r.cltLock.RLock() + client := r.client + r.cltLock.RUnlock() + return client } func (r *kubernetesRegistry) SetClient(client *kubernetes.Client) { + r.cltLock.Lock() r.client = client -} -func (r *kubernetesRegistry) ClientLock() *sync.Mutex { - return &r.cltLock + r.cltLock.Unlock() } func (r *kubernetesRegistry) CloseAndNilClient() { @@ -154,7 +160,7 @@ func newKubernetesRegistry(url *common.URL) (registry.Registry, error) { } r.WaitGroup().Add(1) - go kubernetes.HandleClientRestart(r) + go r.HandleClientRestart() r.InitListeners() logger.Debugf("the kubernetes registry started") @@ -178,7 +184,56 @@ func newMockKubernetesRegistry( return nil, perrors.WithMessage(err, "new mock client") } r.WaitGroup().Add(1) //zk client start successful, then wg +1 - go kubernetes.HandleClientRestart(r) + go r.HandleClientRestart() r.InitListeners() return r, nil } + +func (r *kubernetesRegistry) HandleClientRestart() { + + var ( + err error + failTimes int + ) + + defer r.WaitGroup() +LOOP: + for { + select { + case <-r.Done(): + logger.Warnf("(KubernetesProviderRegistry)reconnectKubernetes goroutine exit now...") + break LOOP + // re-register all services + case <-r.Client().Done(): + r.Client().Close() + r.SetClient(nil) + + // try to connect to kubernetes, + failTimes = 0 + for { + select { + case <-r.Done(): + logger.Warnf("(KubernetesProviderRegistry)reconnectKubernetes Registry goroutine exit now...") + break LOOP + case <-getty.GetTimeWheel().After(timeSecondDuration(failTimes * ConnDelay)): // avoid connect frequent + } + err = kubernetes.ValidateClient(r) + logger.Infof("Kubernetes ProviderRegistry.validateKubernetesClient = error{%#v}", perrors.WithStack(err)) + + if err == nil { + if r.RestartCallBack() { + break + } + } + failTimes++ + if MaxFailTimes <= failTimes { + failTimes = MaxFailTimes + } + } + } + } +} + +func timeSecondDuration(sec int) time.Duration { + return time.Duration(sec) * time.Second +} diff --git a/registry/kubernetes/registry_test.go b/registry/kubernetes/registry_test.go index 22873606b6..a01af167c3 100644 --- a/registry/kubernetes/registry_test.go +++ b/registry/kubernetes/registry_test.go @@ -46,6 +46,7 @@ func (s *KubernetesRegistryTestSuite) TestRegister() { if err != nil { t.Fatal(err) } + r.WaitGroup().Done() } func (s *KubernetesRegistryTestSuite) TestSubscribe() { @@ -75,6 +76,8 @@ func (s *KubernetesRegistryTestSuite) TestSubscribe() { } t.Logf("got event %s", serviceEvent) + + r.WaitGroup().Done() } func (s *KubernetesRegistryTestSuite) TestConsumerDestroy() { @@ -85,12 +88,12 @@ func (s *KubernetesRegistryTestSuite) TestConsumerDestroy() { url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) - _, err := r.DoSubscribe(&url) + listener, err := r.DoSubscribe(&url) if err != nil { t.Fatal(err) } - //listener.Close() + listener.Close() time.Sleep(1e9) r.Destroy() @@ -108,7 +111,8 @@ func (s *KubernetesRegistryTestSuite) TestProviderDestroy() { err := r.Register(url) assert.NoError(t, err) - //listener.Close() + r.WaitGroup().Done() + time.Sleep(1e9) r.Destroy() assert.Equal(t, false, r.IsAvailable()) diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index 605a05fb5f..43b8dd4ec3 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -157,7 +157,7 @@ func newMockClient(namespace string, mockClientGenerator func() (kubernetes.Inte return nil, perrors.WithMessage(err, "watch pods") } - logger.Info("init kubernetes registry success") + logger.Infof("init kubernetes registry client success @namespace = %q @Podname = %q", namespace, c.currentPod.Name) return c, nil } @@ -210,7 +210,7 @@ func newClient(namespace string) (*Client, error) { return nil, perrors.WithMessage(err, "watch pods") } - logger.Info("init kubernetes registry success") + logger.Infof("init kubernetes registry client success @namespace = %q @Podname = %q", namespace, c.currentPod.Name) return c, nil } @@ -306,7 +306,7 @@ func (c *Client) watchPodsLoop() { }() for { - + onceWatch: wc, err := c.rawClient.CoreV1().Pods(c.ns).Watch(metav1.ListOptions{ LabelSelector: fields.OneTermEqualSelector(DubboIOLabelKey, DubboIOLabelValue).String(), Watch: true, @@ -362,7 +362,6 @@ func (c *Client) watchPodsLoop() { go c.handleWatchedPodEvent(p, event.Type) } } - onceWatch: } } @@ -398,7 +397,7 @@ func (c *Client) handleWatchedPodEvent(p *v1.Pod, eventType watch.EventType) { return } - logger.Debugf("prepare to put object (%#v) to kuberentes-watcherSet", o) + logger.Debugf("prepare to put object (%#v) to kubernetes-watcherSet", o) if err := c.watcherSet.Put(o); err != nil { logger.Errorf("put (%#v) to cache watcherSet: %v ", o, err) @@ -454,7 +453,7 @@ func (c *Client) readCurrentPod() (*v1.Pod, error) { } // Create -// create k/v pair in storage +// create k/v pair in watcher-set func (c *Client) Create(k, v string) error { // 1. accord old pod && (k, v) assemble new pod dubbo annotion v @@ -671,26 +670,10 @@ func (c *Client) Close() { // validate the kubernetes client func ValidateClient(container clientFacade) error { - lock := container.ClientLock() - lock.Lock() - defer lock.Unlock() + client := container.Client() // new Client - if container.Client() == nil { - ns, err := getCurrentNameSpace() - if err != nil { - return perrors.WithMessage(err, "get current namespace") - } - newClient, err := newClient(ns) - if err != nil { - logger.Warnf("new kubernetes client (namespace{%s}: %v)", ns, err) - return perrors.WithMessagef(err, "new kubernetes client (:%+v)", ns) - } - container.SetClient(newClient) - } - - if !container.Client().Valid() { - + if client == nil || client.Valid() { ns, err := getCurrentNameSpace() if err != nil { return perrors.WithMessage(err, "get current namespace") diff --git a/remoting/kubernetes/facade.go b/remoting/kubernetes/facade.go index 17a4f41767..dd15c918b4 100644 --- a/remoting/kubernetes/facade.go +++ b/remoting/kubernetes/facade.go @@ -17,78 +17,7 @@ package kubernetes -import ( - "sync" -) - -import ( - "github.com/dubbogo/getty" - perrors "github.com/pkg/errors" -) - -import ( - "github.com/apache/dubbo-go/common" - "github.com/apache/dubbo-go/common/logger" -) - -const ( - ConnDelay = 3 - MaxFailTimes = 15 -) - type clientFacade interface { Client() *Client SetClient(*Client) - ClientLock() *sync.Mutex - WaitGroup() *sync.WaitGroup //for wait group control, etcd client listener & etcd client container - Done() chan struct{} //for etcd client control - RestartCallBack() bool - common.Node -} - -func HandleClientRestart(r clientFacade) { - - var ( - err error - failTimes int - ) - - defer r.WaitGroup().Done() -LOOP: - for { - select { - case <-r.Done(): - logger.Warnf("(KubernetesProviderRegistry)reconnectKubernetes goroutine exit now...") - break LOOP - // re-register all services - case <-r.Client().Done(): - r.ClientLock().Lock() - r.Client().Close() - r.SetClient(nil) - r.ClientLock().Unlock() - - // try to connect to kubernetes, - failTimes = 0 - for { - select { - case <-r.Done(): - logger.Warnf("(KubernetesProviderRegistry)reconnectKubernetes Registry goroutine exit now...") - break LOOP - case <-getty.GetTimeWheel().After(timeSecondDuration(failTimes * ConnDelay)): // avoid connect frequent - } - err = ValidateClient(r) - logger.Infof("Kubernetes ProviderRegistry.validateKubernetesClient = error{%#v}", perrors.WithStack(err)) - - if err == nil { - if r.RestartCallBack() { - break - } - } - failTimes++ - if MaxFailTimes <= failTimes { - failTimes = MaxFailTimes - } - } - } - } } diff --git a/remoting/kubernetes/facade_test.go b/remoting/kubernetes/facade_test.go index cb9f92723f..024264ffde 100644 --- a/remoting/kubernetes/facade_test.go +++ b/remoting/kubernetes/facade_test.go @@ -24,15 +24,10 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" ) -import ( - "github.com/apache/dubbo-go/common" -) type mockFacade struct { client *Client cltLock sync.Mutex - wg sync.WaitGroup - URL *common.URL done chan struct{} } @@ -44,35 +39,6 @@ func (r *mockFacade) SetClient(client *Client) { r.client = client } -func (r *mockFacade) ClientLock() *sync.Mutex { - return &r.cltLock -} - -func (r *mockFacade) WaitGroup() *sync.WaitGroup { - return &r.wg -} - -func (r *mockFacade) Done() chan struct{} { - return r.done -} - -func (r *mockFacade) GetUrl() common.URL { - return *r.URL -} - -func (r *mockFacade) Destroy() { - close(r.done) - r.wg.Wait() -} - -func (r *mockFacade) RestartCallBack() bool { - return true -} - -func (r *mockFacade) IsAvailable() bool { - return true -} - func (s *KubernetesClientTestSuite) Test_Facade() { t := s.T() @@ -91,12 +57,12 @@ func (s *KubernetesClientTestSuite) Test_Facade() { t.Fatal(err) } - url, _ := common.NewURL("mock://127.0.0.1") m := &mockFacade{ client: mockClient, - URL: &url, } - go HandleClientRestart(m) + if err := ValidateClient(m); err == nil { + t.Fatal("out of cluster should err") + } mockClient.Close() } diff --git a/remoting/kubernetes/listener.go b/remoting/kubernetes/listener.go index 575734ab30..c0f9e9f119 100644 --- a/remoting/kubernetes/listener.go +++ b/remoting/kubernetes/listener.go @@ -19,7 +19,6 @@ package kubernetes import ( "sync" - "time" ) import ( @@ -157,10 +156,6 @@ func (l *EventListener) ListenServiceNodeEventWithPrefix(prefix string, listener } } -func timeSecondDuration(sec int) time.Duration { - return time.Duration(sec) * time.Second -} - // this func is invoked by kubernetes ConsumerRegistry::Registry/ kubernetes ConsumerRegistry::get/kubernetes ConsumerRegistry::getListener // registry.go:Listen -> listenServiceEvent -> listenDirEvent -> ListenServiceNodeEvent // | From 603c1b93eccf8dc6732ce9aa91f08b109b34b4da Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 21:09:22 +0800 Subject: [PATCH 37/61] Fix wg bug, add(1) out of goroutine --- remoting/etcdv3/listener.go | 5 ++--- remoting/kubernetes/listener.go | 5 +++-- remoting/zookeeper/listener.go | 5 +++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/remoting/etcdv3/listener.go b/remoting/etcdv3/listener.go index e69f726987..e3cb74e4f6 100644 --- a/remoting/etcdv3/listener.go +++ b/remoting/etcdv3/listener.go @@ -53,7 +53,6 @@ func NewEventListener(client *Client) *EventListener { // this method will return true when spec key deleted, // this method will return false when deep layer connection lose func (l *EventListener) ListenServiceNodeEvent(key string, listener ...remoting.DataListener) bool { - l.wg.Add(1) defer l.wg.Done() for { wc, err := l.client.Watch(key) @@ -138,8 +137,6 @@ func (l *EventListener) handleEvents(event *clientv3.Event, listeners ...remotin // ListenServiceNodeEventWithPrefix Listen on a set of key with spec prefix func (l *EventListener) ListenServiceNodeEventWithPrefix(prefix string, listener ...remoting.DataListener) { - - l.wg.Add(1) defer l.wg.Done() for { wc, err := l.client.WatchWithPrefix(prefix) @@ -217,12 +214,14 @@ func (l *EventListener) ListenServiceEvent(key string, listener remoting.DataLis } logger.Infof("listen dubbo provider key{%s} event and wait to get all provider etcdv3 nodes", key) + l.wg.Add(1) go func(key string, listener remoting.DataListener) { l.ListenServiceNodeEventWithPrefix(key, listener) logger.Warnf("listenDirEvent(key{%s}) goroutine exit now", key) }(key, listener) logger.Infof("listen dubbo service key{%s}", key) + l.wg.Add(1) go func(key string) { if l.ListenServiceNodeEvent(key) { listener.DataChange(remoting.Event{Path: key, Action: remoting.EventTypeDel}) diff --git a/remoting/kubernetes/listener.go b/remoting/kubernetes/listener.go index c0f9e9f119..87deeb38c9 100644 --- a/remoting/kubernetes/listener.go +++ b/remoting/kubernetes/listener.go @@ -48,7 +48,6 @@ func NewEventListener(client *Client) *EventListener { // this method will return true when spec key deleted, // this method will return false when deep layer connection lose func (l *EventListener) ListenServiceNodeEvent(key string, listener ...remoting.DataListener) bool { - l.wg.Add(1) defer l.wg.Done() for { wc, done, err := l.client.Watch(key) @@ -124,7 +123,6 @@ func (l *EventListener) handleEvents(event *WatcherEvent, listeners ...remoting. // Listen on a set of key with spec prefix func (l *EventListener) ListenServiceNodeEventWithPrefix(prefix string, listener ...remoting.DataListener) { - l.wg.Add(1) defer l.wg.Done() for { wc, done, err := l.client.WatchWithPrefix(prefix) @@ -197,12 +195,15 @@ func (l *EventListener) ListenServiceEvent(key string, listener remoting.DataLis } logger.Infof("listen dubbo provider key{%s} event and wait to get all provider from kubernetes-watcherSet", key) + + l.wg.Add(1) go func(key string, listener remoting.DataListener) { l.ListenServiceNodeEventWithPrefix(key, listener) logger.Warnf("listenDirEvent(key{%s}) goroutine exit now", key) }(key, listener) logger.Infof("listen dubbo service key{%s}", key) + l.wg.Add(1) go func(key string) { if l.ListenServiceNodeEvent(key) { listener.DataChange(remoting.Event{Path: key, Action: remoting.EventTypeDel}) diff --git a/remoting/zookeeper/listener.go b/remoting/zookeeper/listener.go index 77aa05ee9e..6c8b1720a7 100644 --- a/remoting/zookeeper/listener.go +++ b/remoting/zookeeper/listener.go @@ -59,7 +59,6 @@ func (l *ZkEventListener) SetClient(client *ZookeeperClient) { // ListenServiceNodeEvent ... func (l *ZkEventListener) ListenServiceNodeEvent(zkPath string, listener ...remoting.DataListener) bool { - l.wg.Add(1) defer l.wg.Done() var zkEvent zk.Event for { @@ -145,6 +144,7 @@ func (l *ZkEventListener) handleZkNodeEvent(zkPath string, children []string, li continue } // listen l service node + l.wg.Add(1) go func(node string, zkPath string, listener remoting.DataListener) { logger.Infof("delete zkNode{%s}", node) if l.ListenServiceNodeEvent(node, listener) { @@ -174,7 +174,6 @@ func (l *ZkEventListener) handleZkNodeEvent(zkPath string, children []string, li } func (l *ZkEventListener) listenDirEvent(zkPath string, listener remoting.DataListener) { - l.wg.Add(1) defer l.wg.Done() var ( @@ -261,6 +260,7 @@ func (l *ZkEventListener) listenDirEvent(zkPath string, listener remoting.DataLi //if zkPath is end of "providers/ & consumers/" we do not listen children dir if strings.LastIndex(zkPath, constant.PROVIDER_CATEGORY) == -1 && strings.LastIndex(zkPath, constant.CONSUMER_CATEGORY) == -1 { + l.wg.Add(1) go func(zkPath string, listener remoting.DataListener) { l.listenDirEvent(zkPath, listener) logger.Warnf("listenDirEvent(zkPath{%s}) goroutine exit now", zkPath) @@ -292,6 +292,7 @@ func timeSecondDuration(sec int) time.Duration { // --------> ListenServiceNodeEvent func (l *ZkEventListener) ListenServiceEvent(zkPath string, listener remoting.DataListener) { logger.Infof("listen dubbo path{%s}", zkPath) + l.wg.Add(1) go func(zkPath string, listener remoting.DataListener) { l.listenDirEvent(zkPath, listener) logger.Warnf("listenDirEvent(zkPath{%s}) goroutine exit now", zkPath) From 8773a7d3160fd6da27230628f5d9d3ba6176979a Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 21:24:32 +0800 Subject: [PATCH 38/61] Fix interest url slice-> map --- registry/etcdv3/listener.go | 31 ++++++++++++++++--------------- registry/kubernetes/listener.go | 33 ++++++++++++++++++--------------- registry/zookeeper/listener.go | 26 +++++++++++++++++--------- 3 files changed, 51 insertions(+), 39 deletions(-) diff --git a/registry/etcdv3/listener.go b/registry/etcdv3/listener.go index 6fa84126f8..61c71ee48e 100644 --- a/registry/etcdv3/listener.go +++ b/registry/etcdv3/listener.go @@ -34,17 +34,20 @@ import ( ) type dataListener struct { - interestedURL []*common.URL + interestedURL map[string]*common.URL listener config_center.ConfigurationListener } // NewRegistryDataListener ... func NewRegistryDataListener(listener config_center.ConfigurationListener) *dataListener { - return &dataListener{listener: listener, interestedURL: []*common.URL{}} + return &dataListener{listener: listener, interestedURL: make(map[string]*common.URL, 16)} } func (l *dataListener) AddInterestedURL(url *common.URL) { - l.interestedURL = append(l.interestedURL, url) + if _, ok := l.interestedURL[url.String()]; ok { + return + } + l.interestedURL[url.String()] = url } func (l *dataListener) DataChange(eventType remoting.Event) bool { @@ -56,20 +59,18 @@ func (l *dataListener) DataChange(eventType remoting.Event) bool { return false } - for _, v := range l.interestedURL { - if serviceURL.URLEqual(*v) { - l.listener.Process( - &config_center.ConfigChangeEvent{ - Key: eventType.Path, - Value: serviceURL, - ConfigType: eventType.Action, - }, - ) - return true - } + if _, ok := l.interestedURL[serviceURL.String()]; !ok { + return false } - return false + l.listener.Process( + &config_center.ConfigChangeEvent{ + Key: eventType.Path, + Value: serviceURL, + ConfigType: eventType.Action, + }, + ) + return true } type configurationListener struct { diff --git a/registry/kubernetes/listener.go b/registry/kubernetes/listener.go index c1d4f70fed..7837233c18 100644 --- a/registry/kubernetes/listener.go +++ b/registry/kubernetes/listener.go @@ -34,17 +34,21 @@ import ( ) type dataListener struct { - interestedURL []*common.URL + interestedURL map[string]*common.URL listener config_center.ConfigurationListener } // NewRegistryDataListener ... func NewRegistryDataListener(listener config_center.ConfigurationListener) *dataListener { - return &dataListener{listener: listener} + return &dataListener{listener: listener, interestedURL: make(map[string]*common.URL, 16)} } func (l *dataListener) AddInterestedURL(url *common.URL) { - l.interestedURL = append(l.interestedURL, url) + + if _, ok := l.interestedURL[url.String()]; ok { + return + } + l.interestedURL[url.String()] = url } func (l *dataListener) DataChange(eventType remoting.Event) bool { @@ -56,20 +60,19 @@ func (l *dataListener) DataChange(eventType remoting.Event) bool { return false } - for _, v := range l.interestedURL { - if serviceURL.URLEqual(*v) { - l.listener.Process( - &config_center.ConfigChangeEvent{ - Key: eventType.Path, - Value: serviceURL, - ConfigType: eventType.Action, - }, - ) - return true - } + if _, ok := l.interestedURL[serviceURL.String()]; !ok { + return false } - return false + l.listener.Process( + &config_center.ConfigChangeEvent{ + Key: eventType.Path, + Value: serviceURL, + ConfigType: eventType.Action, + }, + ) + return true + } type configurationListener struct { diff --git a/registry/zookeeper/listener.go b/registry/zookeeper/listener.go index fe8e42db9f..0c46261b39 100644 --- a/registry/zookeeper/listener.go +++ b/registry/zookeeper/listener.go @@ -37,18 +37,21 @@ import ( // RegistryDataListener ... type RegistryDataListener struct { - interestedURL []*common.URL + interestedURL map[string]*common.URL listener config_center.ConfigurationListener } // NewRegistryDataListener ... func NewRegistryDataListener(listener config_center.ConfigurationListener) *RegistryDataListener { - return &RegistryDataListener{listener: listener, interestedURL: []*common.URL{}} + return &RegistryDataListener{listener: listener, interestedURL: make(map[string]*common.URL, 16)} } // AddInterestedURL ... func (l *RegistryDataListener) AddInterestedURL(url *common.URL) { - l.interestedURL = append(l.interestedURL, url) + if _, ok := l.interestedURL[url.String()]; ok { + return + } + l.interestedURL[url.String()] = url } // DataChange ... @@ -65,14 +68,19 @@ func (l *RegistryDataListener) DataChange(eventType remoting.Event) bool { logger.Errorf("Listen NewURL(r{%s}) = error{%v} eventType.Path={%v}", url, err, eventType.Path) return false } - for _, v := range l.interestedURL { - if serviceURL.URLEqual(*v) { - l.listener.Process(&config_center.ConfigChangeEvent{Value: serviceURL, ConfigType: eventType.Action}) - return true - } + + if _, ok := l.interestedURL[serviceURL.String()]; !ok { + return false } - return false + l.listener.Process( + &config_center.ConfigChangeEvent{ + Key: eventType.Path, + Value: serviceURL, + ConfigType: eventType.Action, + }, + ) + return true } // RegistryConfigurationListener ... From 59fe063c0b7c6165081a1559a9d1eb114233b0c5 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 21:33:08 +0800 Subject: [PATCH 39/61] Fix lock scop --- remoting/kubernetes/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index 43b8dd4ec3..3072aee2cc 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -459,8 +459,6 @@ func (c *Client) Create(k, v string) error { // 1. accord old pod && (k, v) assemble new pod dubbo annotion v // 2. get patch data // 3. PATCH the pod - c.lock.Lock() - defer c.lock.Unlock() currentPod, err := c.readCurrentPod() if err != nil { @@ -482,7 +480,9 @@ func (c *Client) Create(k, v string) error { return perrors.WithMessage(err, "patch current pod") } + c.lock.Lock() c.currentPod = updatedPod + c.lock.Unlock() // not update the watcherSet, the watcherSet should be write by the watchPodsLoop return nil } From 168a9789b2cba4308d4aecfc7e5ade0ef3385006 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 21:38:41 +0800 Subject: [PATCH 40/61] Fix kubernetes registry configListener nil condition --- registry/kubernetes/listener.go | 6 +++++- registry/kubernetes/registry.go | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/registry/kubernetes/listener.go b/registry/kubernetes/listener.go index 7837233c18..528ec6486e 100644 --- a/registry/kubernetes/listener.go +++ b/registry/kubernetes/listener.go @@ -38,11 +38,13 @@ type dataListener struct { listener config_center.ConfigurationListener } -// NewRegistryDataListener ... +// NewRegistryDataListener +// new the data listener, the interest url default len 16 func NewRegistryDataListener(listener config_center.ConfigurationListener) *dataListener { return &dataListener{listener: listener, interestedURL: make(map[string]*common.URL, 16)} } +// AddInterestedURL func (l *dataListener) AddInterestedURL(url *common.URL) { if _, ok := l.interestedURL[url.String()]; ok { @@ -51,6 +53,8 @@ func (l *dataListener) AddInterestedURL(url *common.URL) { l.interestedURL[url.String()] = url } +// DataChange +// notify listen, when interest event func (l *dataListener) DataChange(eventType remoting.Event) bool { url := eventType.Path[strings.Index(eventType.Path, "/providers/")+len("/providers/"):] diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index 79f066a9bd..14dd4a3722 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -87,9 +87,13 @@ func (r *kubernetesRegistry) CloseAndNilClient() { } func (r *kubernetesRegistry) CloseListener() { + + r.cltLock.Lock() if r.configListener != nil { r.configListener.Close() } + r.configListener = nil + r.cltLock.Unlock() } func (r *kubernetesRegistry) CreatePath(k string) error { From 92f9ea21747cf49d5403c25fe96f62a8e6967f07 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 21:42:43 +0800 Subject: [PATCH 41/61] Fix zookeeper wg bug --- remoting/zookeeper/listener.go | 1 + 1 file changed, 1 insertion(+) diff --git a/remoting/zookeeper/listener.go b/remoting/zookeeper/listener.go index 6c8b1720a7..eaf259f441 100644 --- a/remoting/zookeeper/listener.go +++ b/remoting/zookeeper/listener.go @@ -249,6 +249,7 @@ func (l *ZkEventListener) listenDirEvent(zkPath string, listener remoting.DataLi continue } logger.Infof("listen dubbo service key{%s}", dubboPath) + l.wg.Add(1) go func(zkPath string, listener remoting.DataListener) { if l.ListenServiceNodeEvent(zkPath) { listener.DataChange(remoting.Event{Path: zkPath, Action: remoting.EventTypeDel}) From 6b04b96689f62a15141766e7ee0fc46c6c9dc5b0 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 22:53:00 +0800 Subject: [PATCH 42/61] Fix create-path and push to test --- registry/kubernetes/listener.go | 26 ++++++++++++++------------ registry/kubernetes/listener_test.go | 5 ----- registry/kubernetes/registry.go | 9 ++------- registry/kubernetes/registry_test.go | 6 +++++- 4 files changed, 21 insertions(+), 25 deletions(-) diff --git a/registry/kubernetes/listener.go b/registry/kubernetes/listener.go index 528ec6486e..d9164e7143 100644 --- a/registry/kubernetes/listener.go +++ b/registry/kubernetes/listener.go @@ -64,19 +64,21 @@ func (l *dataListener) DataChange(eventType remoting.Event) bool { return false } - if _, ok := l.interestedURL[serviceURL.String()]; !ok { - return false + for _, v := range l.interestedURL { + + if serviceURL.URLEqual(*v) { + + l.listener.Process( + &config_center.ConfigChangeEvent{ + Key: eventType.Path, + Value: serviceURL, + ConfigType: eventType.Action, + }, + ) + return true + } } - - l.listener.Process( - &config_center.ConfigChangeEvent{ - Key: eventType.Path, - Value: serviceURL, - ConfigType: eventType.Action, - }, - ) - return true - + return false } type configurationListener struct { diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go index 88e1589020..c9ff626088 100644 --- a/registry/kubernetes/listener_test.go +++ b/registry/kubernetes/listener_test.go @@ -19,8 +19,6 @@ package kubernetes import ( "encoding/json" - "net/http" - _ "net/http/pprof" "os" "strconv" "testing" @@ -239,9 +237,6 @@ func (s *KubernetesRegistryTestSuite) SetupSuite() { if err := os.Setenv(nameSpaceKey, s.currentPod.GetNamespace()); err != nil { t.Fatal(err) } - - go http.ListenAndServe(":6061", nil) - } func (s *KubernetesRegistryTestSuite) TestDataChange() { diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index 14dd4a3722..289a42aa5a 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -97,14 +97,9 @@ func (r *kubernetesRegistry) CloseListener() { } func (r *kubernetesRegistry) CreatePath(k string) error { - var tmpPath string - for _, str := range strings.Split(k, "/")[1:] { - tmpPath = path.Join(tmpPath, "/", str) - if err := r.client.Create(tmpPath, ""); err != nil { - return perrors.WithMessagef(err, "create path %s in kubernetes", tmpPath) - } + if err := r.client.Create(k, ""); err != nil { + return perrors.WithMessagef(err, "create path %s in kubernetes", k) } - return nil } diff --git a/registry/kubernetes/registry_test.go b/registry/kubernetes/registry_test.go index a01af167c3..43a1bd79b6 100644 --- a/registry/kubernetes/registry_test.go +++ b/registry/kubernetes/registry_test.go @@ -38,7 +38,11 @@ func (s *KubernetesRegistryTestSuite) TestRegister() { r := s.initRegistry() defer r.Destroy() - url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) + url, _ := common.NewURL( + "dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", + common.WithParamsValue(constant.CLUSTER_KEY, "mock"), + common.WithMethods([]string{"GetUser", "AddUser"}), + ) err := r.Register(url) assert.NoError(t, err) From 25f366c794852ae1ab69104cde71b83e78b1044b Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 23:02:03 +0800 Subject: [PATCH 43/61] Fix listener slice -> map --- registry/etcdv3/listener.go | 32 +++++++++++++++----------------- registry/kubernetes/listener.go | 12 +++--------- registry/zookeeper/listener.go | 32 +++++++++++++++----------------- 3 files changed, 33 insertions(+), 43 deletions(-) diff --git a/registry/etcdv3/listener.go b/registry/etcdv3/listener.go index 61c71ee48e..da6a46b15c 100644 --- a/registry/etcdv3/listener.go +++ b/registry/etcdv3/listener.go @@ -34,20 +34,17 @@ import ( ) type dataListener struct { - interestedURL map[string]*common.URL + interestedURL []*common.URL listener config_center.ConfigurationListener } // NewRegistryDataListener ... func NewRegistryDataListener(listener config_center.ConfigurationListener) *dataListener { - return &dataListener{listener: listener, interestedURL: make(map[string]*common.URL, 16)} + return &dataListener{listener: listener, interestedURL: make([]*common.URL, 16)} } func (l *dataListener) AddInterestedURL(url *common.URL) { - if _, ok := l.interestedURL[url.String()]; ok { - return - } - l.interestedURL[url.String()] = url + l.interestedURL = append(l.interestedURL, url) } func (l *dataListener) DataChange(eventType remoting.Event) bool { @@ -59,18 +56,19 @@ func (l *dataListener) DataChange(eventType remoting.Event) bool { return false } - if _, ok := l.interestedURL[serviceURL.String()]; !ok { - return false + for _, v := range l.interestedURL { + if serviceURL.URLEqual(*v) { + l.listener.Process( + &config_center.ConfigChangeEvent{ + Key: eventType.Path, + Value: serviceURL, + ConfigType: eventType.Action, + }, + ) + return true + } } - - l.listener.Process( - &config_center.ConfigChangeEvent{ - Key: eventType.Path, - Value: serviceURL, - ConfigType: eventType.Action, - }, - ) - return true + return false } type configurationListener struct { diff --git a/registry/kubernetes/listener.go b/registry/kubernetes/listener.go index d9164e7143..b1593e6fc8 100644 --- a/registry/kubernetes/listener.go +++ b/registry/kubernetes/listener.go @@ -34,23 +34,19 @@ import ( ) type dataListener struct { - interestedURL map[string]*common.URL + interestedURL []*common.URL listener config_center.ConfigurationListener } // NewRegistryDataListener // new the data listener, the interest url default len 16 func NewRegistryDataListener(listener config_center.ConfigurationListener) *dataListener { - return &dataListener{listener: listener, interestedURL: make(map[string]*common.URL, 16)} + return &dataListener{listener: listener, interestedURL: make([]*common.URL, 16)} } // AddInterestedURL func (l *dataListener) AddInterestedURL(url *common.URL) { - - if _, ok := l.interestedURL[url.String()]; ok { - return - } - l.interestedURL[url.String()] = url + l.interestedURL = append(l.interestedURL, url) } // DataChange @@ -65,9 +61,7 @@ func (l *dataListener) DataChange(eventType remoting.Event) bool { } for _, v := range l.interestedURL { - if serviceURL.URLEqual(*v) { - l.listener.Process( &config_center.ConfigChangeEvent{ Key: eventType.Path, diff --git a/registry/zookeeper/listener.go b/registry/zookeeper/listener.go index 0c46261b39..6b7f7aa0e3 100644 --- a/registry/zookeeper/listener.go +++ b/registry/zookeeper/listener.go @@ -37,21 +37,18 @@ import ( // RegistryDataListener ... type RegistryDataListener struct { - interestedURL map[string]*common.URL + interestedURL []*common.URL listener config_center.ConfigurationListener } // NewRegistryDataListener ... func NewRegistryDataListener(listener config_center.ConfigurationListener) *RegistryDataListener { - return &RegistryDataListener{listener: listener, interestedURL: make(map[string]*common.URL, 16)} + return &RegistryDataListener{listener: listener, interestedURL: make([]*common.URL, 16)} } // AddInterestedURL ... func (l *RegistryDataListener) AddInterestedURL(url *common.URL) { - if _, ok := l.interestedURL[url.String()]; ok { - return - } - l.interestedURL[url.String()] = url + l.interestedURL = append(l.interestedURL, url) } // DataChange ... @@ -69,18 +66,19 @@ func (l *RegistryDataListener) DataChange(eventType remoting.Event) bool { return false } - if _, ok := l.interestedURL[serviceURL.String()]; !ok { - return false + for _, v := range l.interestedURL { + if serviceURL.URLEqual(*v) { + l.listener.Process( + &config_center.ConfigChangeEvent{ + Key: eventType.Path, + Value: serviceURL, + ConfigType: eventType.Action, + }, + ) + return true + } } - - l.listener.Process( - &config_center.ConfigChangeEvent{ - Key: eventType.Path, - Value: serviceURL, - ConfigType: eventType.Action, - }, - ) - return true + return false } // RegistryConfigurationListener ... From 5b87f8ec34f08d9e047958983e6aa72cfdf0b9db Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 23:08:18 +0800 Subject: [PATCH 44/61] Fix missing protocol scheme bug --- registry/etcdv3/listener.go | 7 ++++++- registry/kubernetes/listener.go | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/registry/etcdv3/listener.go b/registry/etcdv3/listener.go index da6a46b15c..2f2172d5fc 100644 --- a/registry/etcdv3/listener.go +++ b/registry/etcdv3/listener.go @@ -49,7 +49,12 @@ func (l *dataListener) AddInterestedURL(url *common.URL) { func (l *dataListener) DataChange(eventType remoting.Event) bool { - url := eventType.Path[strings.Index(eventType.Path, "/providers/")+len("/providers/"):] + index := strings.Index(eventType.Path, "/providers/") + if index == -1 { + logger.Warnf("Listen with no url, event.path={%v}", eventType.Path) + return false + } + url := eventType.Path[index+len("/providers/"):] serviceURL, err := common.NewURL(url) if err != nil { logger.Warnf("Listen NewURL(r{%s}) = error{%v}", eventType.Path, err) diff --git a/registry/kubernetes/listener.go b/registry/kubernetes/listener.go index b1593e6fc8..6527eebbb0 100644 --- a/registry/kubernetes/listener.go +++ b/registry/kubernetes/listener.go @@ -53,7 +53,12 @@ func (l *dataListener) AddInterestedURL(url *common.URL) { // notify listen, when interest event func (l *dataListener) DataChange(eventType remoting.Event) bool { - url := eventType.Path[strings.Index(eventType.Path, "/providers/")+len("/providers/"):] + index := strings.Index(eventType.Path, "/providers/") + if index == -1 { + logger.Warnf("Listen with no url, event.path={%v}", eventType.Path) + return false + } + url := eventType.Path[index+len("/providers/"):] serviceURL, err := common.NewURL(url) if err != nil { logger.Warnf("Listen NewURL(r{%s}) = error{%v}", eventType.Path, err) From b7af87580718bbe4c3f8c32b3cea695f0b9a30e1 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 23:16:17 +0800 Subject: [PATCH 45/61] Fix nil point, the make slice will import a nil object in slice --- registry/etcdv3/listener.go | 4 ++-- registry/kubernetes/listener.go | 3 +-- registry/zookeeper/listener.go | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/registry/etcdv3/listener.go b/registry/etcdv3/listener.go index 2f2172d5fc..51fdf21f5d 100644 --- a/registry/etcdv3/listener.go +++ b/registry/etcdv3/listener.go @@ -38,9 +38,9 @@ type dataListener struct { listener config_center.ConfigurationListener } -// NewRegistryDataListener ... +// NewRegistryDataListener func NewRegistryDataListener(listener config_center.ConfigurationListener) *dataListener { - return &dataListener{listener: listener, interestedURL: make([]*common.URL, 16)} + return &dataListener{listener: listener} } func (l *dataListener) AddInterestedURL(url *common.URL) { diff --git a/registry/kubernetes/listener.go b/registry/kubernetes/listener.go index 6527eebbb0..f8869fea7b 100644 --- a/registry/kubernetes/listener.go +++ b/registry/kubernetes/listener.go @@ -39,9 +39,8 @@ type dataListener struct { } // NewRegistryDataListener -// new the data listener, the interest url default len 16 func NewRegistryDataListener(listener config_center.ConfigurationListener) *dataListener { - return &dataListener{listener: listener, interestedURL: make([]*common.URL, 16)} + return &dataListener{listener: listener} } // AddInterestedURL diff --git a/registry/zookeeper/listener.go b/registry/zookeeper/listener.go index 6b7f7aa0e3..bef1760e04 100644 --- a/registry/zookeeper/listener.go +++ b/registry/zookeeper/listener.go @@ -43,7 +43,7 @@ type RegistryDataListener struct { // NewRegistryDataListener ... func NewRegistryDataListener(listener config_center.ConfigurationListener) *RegistryDataListener { - return &RegistryDataListener{listener: listener, interestedURL: make([]*common.URL, 16)} + return &RegistryDataListener{listener: listener} } // AddInterestedURL ... From 1ed3e919b6496b458fbefa3a7fd706efc6bba8c8 Mon Sep 17 00:00:00 2001 From: scott Date: Sun, 15 Mar 2020 23:30:02 +0800 Subject: [PATCH 46/61] Fix go.sum conflict --- CHANGE.md | 28 ++ README.md | 15 +- README_CN.md | 17 +- before_ut.bat | 6 +- common/constant/default.go | 2 + common/constant/key.go | 43 +- common/constant/time.go | 2 + common/extension/auth.go | 6 + common/extension/config_reader.go | 50 ++ common/extension/rest_client.go | 37 ++ common/extension/rest_server.go | 37 ++ common/proxy/proxy.go | 8 + common/yaml/testdata/config.yml | 7 + common/yaml/yaml.go | 50 ++ common/yaml/yaml_test.go | 58 +++ config/base_config.go | 29 +- config/base_config_test.go | 11 - config/condition_router_config.go | 3 +- config/config_loader.go | 30 ++ config/consumer_config.go | 14 +- config/generic_service.go | 4 +- config/interfaces/config_reader.go | 9 + config/provider_config.go | 12 +- config_center/nacos/client.go | 2 +- config_center/nacos/facade.go | 9 +- config_center/parser/configuration_parser.go | 12 +- .../apache-release-procedure-20200306.md | 448 ++++++++++++++++++ doc/apache/release_note.md | 11 + go.mod | 5 +- go.sum | 22 +- protocol/grpc/client.go | 8 +- protocol/grpc/server.go | 8 +- .../rest/client/client_impl/resty_client.go | 85 ++++ protocol/rest/client/rest_client.go | 43 ++ .../rest/config/reader/rest_config_reader.go | 158 ++++++ .../config/reader/rest_config_reader_test.go | 50 ++ .../reader/testdata/consumer_config.yml | 74 +++ .../reader/testdata/provider_config.yml | 88 ++++ protocol/rest/config/rest_config.go | 153 ++++++ protocol/rest/rest_exporter.go | 49 ++ protocol/rest/rest_invoker.go | 109 +++++ protocol/rest/rest_invoker_test.go | 196 ++++++++ protocol/rest/rest_protocol.go | 156 ++++++ protocol/rest/rest_protocol_test.go | 186 ++++++++ protocol/rest/server/rest_server.go | 31 ++ .../server/server_impl/go_restful_server.go | 311 ++++++++++++ registry/base_registry.go | 46 +- 47 files changed, 2624 insertions(+), 114 deletions(-) create mode 100644 common/extension/config_reader.go create mode 100644 common/extension/rest_client.go create mode 100644 common/extension/rest_server.go create mode 100644 common/yaml/testdata/config.yml create mode 100644 common/yaml/yaml.go create mode 100644 common/yaml/yaml_test.go create mode 100644 config/interfaces/config_reader.go create mode 100644 doc/apache/apache-release-procedure-20200306.md create mode 100644 doc/apache/release_note.md create mode 100644 protocol/rest/client/client_impl/resty_client.go create mode 100644 protocol/rest/client/rest_client.go create mode 100644 protocol/rest/config/reader/rest_config_reader.go create mode 100644 protocol/rest/config/reader/rest_config_reader_test.go create mode 100644 protocol/rest/config/reader/testdata/consumer_config.yml create mode 100644 protocol/rest/config/reader/testdata/provider_config.yml create mode 100644 protocol/rest/config/rest_config.go create mode 100644 protocol/rest/rest_exporter.go create mode 100644 protocol/rest/rest_invoker.go create mode 100644 protocol/rest/rest_invoker_test.go create mode 100644 protocol/rest/rest_protocol.go create mode 100644 protocol/rest/rest_protocol_test.go create mode 100644 protocol/rest/server/rest_server.go create mode 100644 protocol/rest/server/server_impl/go_restful_server.go diff --git a/CHANGE.md b/CHANGE.md index 9864601a37..8b16870ef8 100644 --- a/CHANGE.md +++ b/CHANGE.md @@ -1,6 +1,34 @@ # Release Notes --- +## 1.4.0 +### New Features + +- [Condition router](https://github.com/apache/dubbo-go/pull/294) +- [Context support](https://github.com/apache/dubbo-go/pull/330) +- [Opentracing & transfer context end to end for jsonrpc protocol](https://github.com/apache/dubbo-go/pull/335) +- [Opentracing & transfer context end to end for dubbo protocol](https://github.com/apache/dubbo-go/pull/344) +- [Nacos config center](https://github.com/apache/dubbo-go/pull/357) +- [Prometheus support](https://github.com/apache/dubbo-go/pull/342) +- [Support sign and auth for request](https://github.com/apache/dubbo-go/pull/323) +- [Healthy instance first router](https://github.com/apache/dubbo-go/pull/389) +- [User can add attachments for dubbo protocol](https://github.com/apache/dubbo-go/pull/398) +- [K8s as registry](https://github.com/apache/dubbo-go/pull/400) +- [Rest protocol](https://github.com/apache/dubbo-go/pull/352) + +### Enhancement + +- [Reduce the scope of lock in zk listener](https://github.com/apache/dubbo-go/pull/346) +- [Trace error of getGettyRpcClient](https://github.com/apache/dubbo-go/pull/384) +- [Refactor to add base_registry](https://github.com/apache/dubbo-go/pull/348) +- [Do not listen to directory event if zkPath ends with providers/ or consumers/](https://github.com/apache/dubbo-go/pull/359) + +### Bugfixes + +- [Handle the panic when invoker was destroyed](https://github.com/apache/dubbo-go/pull/358) +- [HessianCodec failed to check package header length](https://github.com/apache/dubbo-go/pull/381) + + ## 1.3.0 ### New Features diff --git a/README.md b/README.md index 1dde951d35..c74769827b 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ [![Build Status](https://travis-ci.org/apache/dubbo-go.svg?branch=master)](https://travis-ci.org/apache/dubbo-go) [![codecov](https://codecov.io/gh/apache/dubbo-go/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/dubbo-go) [![go.dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/apache/dubbo-go?tab=doc) +[![Go Report Card](https://goreportcard.com/badge/github.com/apache/dubbo-go)](https://goreportcard.com/report/github.com/apache/dubbo-go) +![license](https://img.shields.io/badge/license-Apache--2.0-green.svg) --- Apache Dubbo Go Implementation. @@ -50,16 +52,22 @@ Finished List: * Dubbo * Jsonrpc2.0 * [gRPC](https://github.com/apache/dubbo-go/pull/311) + * [RESTful](https://github.com/apache/dubbo-go/pull/352) + +- Router + * [Conditional router](https://github.com/apache/dubbo-go/pull/294) - Registry * ZooKeeper * [etcd v3](https://github.com/apache/dubbo-go/pull/148) * [nacos](https://github.com/apache/dubbo-go/pull/151) * [consul](https://github.com/apache/dubbo-go/pull/121) + * [k8s](https://github.com/apache/dubbo-go/pull/400) - Dynamic Configure Center & Service Management Configurator * Zookeeper * [apollo](https://github.com/apache/dubbo-go/pull/250) + * [nacos](https://github.com/apache/dubbo-go/pull/357) - Cluster Strategy * Failover @@ -86,6 +94,10 @@ Finished List: - Invoke * [generic invoke](https://github.com/apache/dubbo-go/pull/122) + +- Monitor + * Opentracing API + * Prometheus - Others: * start check @@ -97,9 +109,8 @@ Finished List: Working List: -- Registry: k8s - Metadata Center (dubbo v2.7.x) -- Metrics: Opentracing/Promethus(dubbo v2.7.x) +- Service Discovery (dubbo v2.7.x) You can know more about dubbo-go by its [roadmap](https://github.com/apache/dubbo-go/wiki/Roadmap). diff --git a/README_CN.md b/README_CN.md index ade924e7a9..d01e5efbdf 100644 --- a/README_CN.md +++ b/README_CN.md @@ -3,6 +3,8 @@ [![Build Status](https://travis-ci.org/apache/dubbo-go.svg?branch=master)](https://travis-ci.org/apache/dubbo-go) [![codecov](https://codecov.io/gh/apache/dubbo-go/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/dubbo-go) [![go.dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/apache/dubbo-go?tab=doc) +[![Go Report Card](https://goreportcard.com/badge/github.com/apache/dubbo-go)](https://goreportcard.com/report/github.com/apache/dubbo-go) +![license](https://img.shields.io/badge/license-Apache--2.0-green.svg) --- Apache Dubbo Go 语言实现 @@ -49,16 +51,22 @@ Apache License, Version 2.0 * Dubbo * Jsonrpc2.0 * [gRPC](https://github.com/apache/dubbo-go/pull/311) - + * [RESTful](https://github.com/apache/dubbo-go/pull/352) + +- 路由器 + * [Conditional router](https://github.com/apache/dubbo-go/pull/294) + - 注册中心 * ZooKeeper * [etcd v3](https://github.com/apache/dubbo-go/pull/148) * [nacos](https://github.com/apache/dubbo-go/pull/151) * [consul](https://github.com/apache/dubbo-go/pull/121) + * [k8s](https://github.com/apache/dubbo-go/pull/400) - 动态配置中心与服务治理配置器 * Zookeeper * [apollo](https://github.com/apache/dubbo-go/pull/250) + * [nacos](https://github.com/apache/dubbo-go/pull/357) - 集群策略 * Failover @@ -84,6 +92,10 @@ Apache License, Version 2.0 - 调用 * [泛化调用](https://github.com/apache/dubbo-go/pull/122) + +- 监控 + * Opentracing API + * Prometheus - 其他功能支持: * 启动时检查 @@ -95,9 +107,8 @@ Apache License, Version 2.0 开发中列表: -- 注册中心: k8s - 元数据中心 (dubbo v2.7.x) -- Metrics: Opentracing/Promethus(dubbo v2.7.x) +- 服务发现 (dubbo v2.7.x) 你可以通过访问 [roadmap](https://github.com/apache/dubbo-go/wiki/Roadmap) 知道更多关于 dubbo-go 的信息。 diff --git a/before_ut.bat b/before_ut.bat index abe7fc250e..5e1c877af2 100644 --- a/before_ut.bat +++ b/before_ut.bat @@ -14,10 +14,10 @@ :: See the License for the specific language governing permissions and :: limitations under the License. -set zkJarName="zookeeper-3.4.9-fatjar.jar" +set zkJarName=zookeeper-3.4.9-fatjar.jar set remoteJarUrl="https://github.com/dubbogo/resources/raw/master/zookeeper-4unitest/contrib/fatjar/%zkJarName%" -set zkJarPath="remoting/zookeeper/zookeeper-4unittest/contrib/fatjar" -set zkJar="%zkJarPath%/%zkJarName%" +set zkJarPath=remoting/zookeeper/zookeeper-4unittest/contrib/fatjar +set zkJar=%zkJarPath%/%zkJarName% if not exist "%zkJar%" ( md %zkJarPath% diff --git a/common/constant/default.go b/common/constant/default.go index 8ed645e84a..3c889158e4 100644 --- a/common/constant/default.go +++ b/common/constant/default.go @@ -41,6 +41,8 @@ const ( DEFAULT_FAILBACK_TIMES = "3" DEFAULT_FAILBACK_TIMES_INT = 3 DEFAULT_FAILBACK_TASKS = 100 + DEFAULT_REST_CLIENT = "resty" + DEFAULT_REST_SERVER = "go-restful" ) const ( diff --git a/common/constant/key.go b/common/constant/key.go index c8a03b3be9..07335bed59 100644 --- a/common/constant/key.go +++ b/common/constant/key.go @@ -174,21 +174,36 @@ const ( ) const ( - CONSUMER_SIGN_FILTER = "sign" - PROVIDER_AUTH_FILTER = "auth" - SERVICE_AUTH_KEY = "auth" - AUTHENTICATOR_KEY = "authenticator" - DEFAULT_AUTHENTICATOR = "accesskeys" - DEFAULT_ACCESS_KEY_STORAGE = "urlstorage" - ACCESS_KEY_STORAGE_KEY = "accessKey.storage" - REQUEST_TIMESTAMP_KEY = "timestamp" - REQUEST_SIGNATURE_KEY = "signature" - AK_KEY = "ak" - SIGNATURE_STRING_FORMAT = "%s#%s#%s#%s" + // name of consumer sign filter + CONSUMER_SIGN_FILTER = "sign" + // name of consumer sign filter + PROVIDER_AUTH_FILTER = "auth" + // name of service filter + SERVICE_AUTH_KEY = "auth" + // key of authenticator + AUTHENTICATOR_KEY = "authenticator" + // name of default authenticator + DEFAULT_AUTHENTICATOR = "accesskeys" + // name of default url storage + DEFAULT_ACCESS_KEY_STORAGE = "urlstorage" + // key of storage + ACCESS_KEY_STORAGE_KEY = "accessKey.storage" + // key of request timestamp + REQUEST_TIMESTAMP_KEY = "timestamp" + // key of request signature + REQUEST_SIGNATURE_KEY = "signature" + // AK key + AK_KEY = "ak" + // signature format + SIGNATURE_STRING_FORMAT = "%s#%s#%s#%s" + // key whether enable signature PARAMTER_SIGNATURE_ENABLE_KEY = "param.sign" - CONSUMER = "consumer" - ACCESS_KEY_ID_KEY = "accessKeyId" - SECRET_ACCESS_KEY_KEY = "secretAccessKey" + // consumer + CONSUMER = "consumer" + // key of access key id + ACCESS_KEY_ID_KEY = "accessKeyId" + // key of secret access key + SECRET_ACCESS_KEY_KEY = "secretAccessKey" ) // HealthCheck Router diff --git a/common/constant/time.go b/common/constant/time.go index be1baaca67..3bb339229b 100644 --- a/common/constant/time.go +++ b/common/constant/time.go @@ -22,5 +22,7 @@ import ( ) var ( + // The value will be 10^6 + // 1ms = 10^6ns MsToNanoRate = int64(time.Millisecond / time.Nanosecond) ) diff --git a/common/extension/auth.go b/common/extension/auth.go index e57e22f660..a35fc509da 100644 --- a/common/extension/auth.go +++ b/common/extension/auth.go @@ -9,10 +9,13 @@ var ( accesskeyStorages = make(map[string]func() filter.AccessKeyStorage) ) +// SetAuthenticator put the fcn into map with name func SetAuthenticator(name string, fcn func() filter.Authenticator) { authenticators[name] = fcn } +// GetAuthenticator find the Authenticator with name +// if not found, it will panic func GetAuthenticator(name string) filter.Authenticator { if authenticators[name] == nil { panic("authenticator for " + name + " is not existing, make sure you have import the package.") @@ -20,10 +23,13 @@ func GetAuthenticator(name string) filter.Authenticator { return authenticators[name]() } +// SetAccesskeyStorages will set the fcn into map with this name func SetAccesskeyStorages(name string, fcn func() filter.AccessKeyStorage) { accesskeyStorages[name] = fcn } +// GetAccesskeyStorages find the storage with the name. +// If not found, it will panic. func GetAccesskeyStorages(name string) filter.AccessKeyStorage { if accesskeyStorages[name] == nil { panic("accesskeyStorages for " + name + " is not existing, make sure you have import the package.") diff --git a/common/extension/config_reader.go b/common/extension/config_reader.go new file mode 100644 index 0000000000..aced5b0281 --- /dev/null +++ b/common/extension/config_reader.go @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package extension + +import ( + "github.com/apache/dubbo-go/config/interfaces" +) + +var ( + configReaders = make(map[string]func() interfaces.ConfigReader) + defaults = make(map[string]string) +) + +// SetConfigReaders set a creator of config reader. +func SetConfigReaders(name string, v func() interfaces.ConfigReader) { + configReaders[name] = v +} + +// GetConfigReaders get a config reader by name. +func GetConfigReaders(name string) interfaces.ConfigReader { + if configReaders[name] == nil { + panic("config reader for " + name + " is not existing, make sure you have imported the package.") + } + return configReaders[name]() +} + +// SetDefaultConfigReader set {name} to default config reader for {module} +func SetDefaultConfigReader(module, name string) { + defaults[module] = name +} + +// GetDefaultConfigReader +func GetDefaultConfigReader() map[string]string { + return defaults +} diff --git a/common/extension/rest_client.go b/common/extension/rest_client.go new file mode 100644 index 0000000000..514d1fdfd2 --- /dev/null +++ b/common/extension/rest_client.go @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package extension + +import ( + "github.com/apache/dubbo-go/protocol/rest/client" +) + +var ( + restClients = make(map[string]func(restOptions *client.RestOptions) client.RestClient, 8) +) + +func SetRestClient(name string, fun func(restOptions *client.RestOptions) client.RestClient) { + restClients[name] = fun +} + +func GetNewRestClient(name string, restOptions *client.RestOptions) client.RestClient { + if restClients[name] == nil { + panic("restClient for " + name + " is not existing, make sure you have import the package.") + } + return restClients[name](restOptions) +} diff --git a/common/extension/rest_server.go b/common/extension/rest_server.go new file mode 100644 index 0000000000..fa8d435a5c --- /dev/null +++ b/common/extension/rest_server.go @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package extension + +import ( + "github.com/apache/dubbo-go/protocol/rest/server" +) + +var ( + restServers = make(map[string]func() server.RestServer, 8) +) + +func SetRestServer(name string, fun func() server.RestServer) { + restServers[name] = fun +} + +func GetNewRestServer(name string) server.RestServer { + if restServers[name] == nil { + panic("restServer for " + name + " is not existing, make sure you have import the package.") + } + return restServers[name]() +} diff --git a/common/proxy/proxy.go b/common/proxy/proxy.go index 6765a810a5..68ba3ff788 100644 --- a/common/proxy/proxy.go +++ b/common/proxy/proxy.go @@ -140,6 +140,14 @@ func (p *Proxy) Implement(v common.RPCService) { inv.SetAttachments(k, value) } + // add user setAttachment + atm := invCtx.Value("attachment") + if m, ok := atm.(map[string]string); ok { + for k, value := range m { + inv.SetAttachments(k, value) + } + } + result := p.invoke.Invoke(invCtx, inv) err = result.Error() diff --git a/common/yaml/testdata/config.yml b/common/yaml/testdata/config.yml new file mode 100644 index 0000000000..b5c2ca8ad1 --- /dev/null +++ b/common/yaml/testdata/config.yml @@ -0,0 +1,7 @@ + +intTest: 11 +booleanTest: false +strTest: "strTest" + +child: + strTest: "childStrTest" \ No newline at end of file diff --git a/common/yaml/yaml.go b/common/yaml/yaml.go new file mode 100644 index 0000000000..7c31d71c35 --- /dev/null +++ b/common/yaml/yaml.go @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package yaml + +import ( + "io/ioutil" + "path" +) + +import ( + perrors "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +// loadYMLConfig Load yml config byte from file +func LoadYMLConfig(confProFile string) ([]byte, error) { + if len(confProFile) == 0 { + return nil, perrors.Errorf("application configure(provider) file name is nil") + } + + if path.Ext(confProFile) != ".yml" { + return nil, perrors.Errorf("application configure file name{%v} suffix must be .yml", confProFile) + } + + return ioutil.ReadFile(confProFile) +} + +// unmarshalYMLConfig Load yml config byte from file , then unmarshal to object +func UnmarshalYMLConfig(confProFile string, out interface{}) ([]byte, error) { + confFileStream, err := LoadYMLConfig(confProFile) + if err != nil { + return confFileStream, perrors.Errorf("ioutil.ReadFile(file:%s) = error:%v", confProFile, perrors.WithStack(err)) + } + return confFileStream, yaml.Unmarshal(confFileStream, out) +} diff --git a/common/yaml/yaml_test.go b/common/yaml/yaml_test.go new file mode 100644 index 0000000000..45eee59048 --- /dev/null +++ b/common/yaml/yaml_test.go @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package yaml + +import ( + "path/filepath" + "testing" +) + +import ( + "github.com/stretchr/testify/assert" +) + +func TestUnmarshalYMLConfig(t *testing.T) { + conPath, err := filepath.Abs("./testdata/config.yml") + assert.NoError(t, err) + c := &Config{} + _, err = UnmarshalYMLConfig(conPath, c) + assert.NoError(t, err) + assert.Equal(t, "strTest", c.StrTest) + assert.Equal(t, 11, c.IntTest) + assert.Equal(t, false, c.BooleanTest) + assert.Equal(t, "childStrTest", c.ChildConfig.StrTest) +} + +func TestUnmarshalYMLConfig_Error(t *testing.T) { + c := &Config{} + _, err := UnmarshalYMLConfig("./testdata/config", c) + assert.Error(t, err) + _, err = UnmarshalYMLConfig("", c) + assert.Error(t, err) +} + +type Config struct { + StrTest string `yaml:"strTest" default:"default" json:"strTest,omitempty" property:"strTest"` + IntTest int `default:"109" yaml:"intTest" json:"intTest,omitempty" property:"intTest"` + BooleanTest bool `yaml:"booleanTest" default:"true" json:"booleanTest,omitempty"` + ChildConfig ChildConfig `yaml:"child" json:"child,omitempty"` +} + +type ChildConfig struct { + StrTest string `default:"strTest" default:"default" yaml:"strTest" json:"strTest,omitempty"` +} diff --git a/config/base_config.go b/config/base_config.go index 6d5ec7e249..93c0ce6a66 100644 --- a/config/base_config.go +++ b/config/base_config.go @@ -18,8 +18,7 @@ package config import ( - "io/ioutil" - "path" + "bytes" "reflect" "strconv" "strings" @@ -27,7 +26,6 @@ import ( import ( perrors "github.com/pkg/errors" - "gopkg.in/yaml.v2" ) import ( @@ -50,6 +48,8 @@ type BaseConfig struct { fatherConfig interface{} MetricConfig *MetricConfig `yaml:"metrics" json:"metrics,omitempty"` + + fileStream *bytes.Buffer } // startConfigCenter will start the config center. @@ -364,27 +364,4 @@ func initializeStruct(t reflect.Type, v reflect.Value) { } } - -} - -// loadYMLConfig Load yml config byte from file -func loadYMLConfig(confProFile string) ([]byte, error) { - if len(confProFile) == 0 { - return nil, perrors.Errorf("application configure(provider) file name is nil") - } - - if path.Ext(confProFile) != ".yml" { - return nil, perrors.Errorf("application configure file name{%v} suffix must be .yml", confProFile) - } - - return ioutil.ReadFile(confProFile) -} - -// unmarshalYMLConfig Load yml config byte from file , then unmarshal to object -func unmarshalYMLConfig(confProFile string, out interface{}) error { - confFileStream, err := loadYMLConfig(confProFile) - if err != nil { - return perrors.Errorf("ioutil.ReadFile(file:%s) = error:%v", confProFile, perrors.WithStack(err)) - } - return yaml.Unmarshal(confFileStream, out) } diff --git a/config/base_config_test.go b/config/base_config_test.go index 6973a4a18b..d16b242092 100644 --- a/config/base_config_test.go +++ b/config/base_config_test.go @@ -18,7 +18,6 @@ package config import ( "fmt" - "path/filepath" "reflect" "testing" ) @@ -518,13 +517,3 @@ func Test_initializeStruct(t *testing.T) { return consumerConfig.References != nil }) } - -func TestUnmarshalYMLConfig(t *testing.T) { - conPath, err := filepath.Abs("./testdata/consumer_config_with_configcenter.yml") - assert.NoError(t, err) - c := &ConsumerConfig{} - assert.NoError(t, unmarshalYMLConfig(conPath, c)) - assert.Equal(t, "default", c.ProxyFactory) - assert.Equal(t, "dubbo.properties", c.ConfigCenterConfig.ConfigFile) - assert.Equal(t, "100ms", c.Connect_Timeout) -} diff --git a/config/condition_router_config.go b/config/condition_router_config.go index a95b2d2b12..87e835108e 100644 --- a/config/condition_router_config.go +++ b/config/condition_router_config.go @@ -25,12 +25,13 @@ import ( "github.com/apache/dubbo-go/cluster/directory" "github.com/apache/dubbo-go/common/extension" "github.com/apache/dubbo-go/common/logger" + "github.com/apache/dubbo-go/common/yaml" ) //RouterInit Load config file to init router config func RouterInit(confRouterFile string) error { fileRouterFactories := extension.GetFileRouterFactories() - bytes, err := loadYMLConfig(confRouterFile) + bytes, err := yaml.LoadYMLConfig(confRouterFile) if err != nil { return perrors.Errorf("ioutil.ReadFile(file:%s) = error:%v", confRouterFile, perrors.WithStack(err)) } diff --git a/config/config_loader.go b/config/config_loader.go index 437f4d7323..c0687d8fc1 100644 --- a/config/config_loader.go +++ b/config/config_loader.go @@ -24,9 +24,14 @@ import ( "time" ) +import ( + perrors "github.com/pkg/errors" +) + import ( "github.com/apache/dubbo-go/common" "github.com/apache/dubbo-go/common/constant" + "github.com/apache/dubbo-go/common/extension" "github.com/apache/dubbo-go/common/logger" ) @@ -78,6 +83,7 @@ func checkApplicationName(config *ApplicationConfig) { // Load Dubbo Init func Load() { + // init router if confRouterFile != "" { if errPro := RouterInit(confRouterFile); errPro != nil { @@ -89,6 +95,18 @@ func Load() { if consumerConfig == nil { logger.Warnf("consumerConfig is nil!") } else { + // init other consumer config + conConfigType := consumerConfig.ConfigType + for key, value := range extension.GetDefaultConfigReader() { + if conConfigType == nil { + if v, ok := conConfigType[key]; ok { + value = v + } + } + if err := extension.GetConfigReaders(value).ReadConsumerConfig(consumerConfig.fileStream); err != nil { + logger.Errorf("ReadConsumerConfig error: %#v for %s", perrors.WithStack(err), value) + } + } metricConfig = consumerConfig.MetricConfig applicationConfig = consumerConfig.ApplicationConfig @@ -150,6 +168,18 @@ func Load() { if providerConfig == nil { logger.Warnf("providerConfig is nil!") } else { + // init other provider config + proConfigType := providerConfig.ConfigType + for key, value := range extension.GetDefaultConfigReader() { + if proConfigType != nil { + if v, ok := proConfigType[key]; ok { + value = v + } + } + if err := extension.GetConfigReaders(value).ReadProviderConfig(providerConfig.fileStream); err != nil { + logger.Errorf("ReadProviderConfig error: %#v for %s", perrors.WithStack(err), value) + } + } // so, you should know that the consumer's config will be override metricConfig = providerConfig.MetricConfig diff --git a/config/consumer_config.go b/config/consumer_config.go index 94da301ce4..1fa68415bf 100644 --- a/config/consumer_config.go +++ b/config/consumer_config.go @@ -18,6 +18,7 @@ package config import ( + "bytes" "time" ) @@ -30,6 +31,7 @@ import ( import ( "github.com/apache/dubbo-go/common/constant" "github.com/apache/dubbo-go/common/logger" + "github.com/apache/dubbo-go/common/yaml" ) ///////////////////////// @@ -57,6 +59,7 @@ type ConsumerConfig struct { ProtocolConf interface{} `yaml:"protocol_conf" json:"protocol_conf,omitempty" property:"protocol_conf"` FilterConf interface{} `yaml:"filter_conf" json:"filter_conf,omitempty" property:"filter_conf" ` ShutdownConfig *ShutdownConfig `yaml:"shutdown_conf" json:"shutdown_conf,omitempty" property:"shutdown_conf" ` + ConfigType map[string]string `yaml:"config_type" json:"config_type,omitempty" property:"config_type"` } // UnmarshalYAML ... @@ -86,13 +89,12 @@ func ConsumerInit(confConFile string) error { if confConFile == "" { return perrors.Errorf("application configure(consumer) file name is nil") } - consumerConfig = &ConsumerConfig{} - err := unmarshalYMLConfig(confConFile, consumerConfig) + fileStream, err := yaml.UnmarshalYMLConfig(confConFile, consumerConfig) if err != nil { - return perrors.Errorf("yaml.Unmarshal() = error:%v", perrors.WithStack(err)) + return perrors.Errorf("unmarshalYmlConfig error %v", perrors.WithStack(err)) } - + consumerConfig.fileStream = bytes.NewBuffer(fileStream) //set method interfaceId & interfaceName for k, v := range consumerConfig.References { //set id for reference @@ -116,6 +118,7 @@ func ConsumerInit(confConFile string) error { } } logger.Debugf("consumer config{%#v}\n", consumerConfig) + return nil } @@ -139,5 +142,6 @@ func configCenterRefreshConsumer() error { return perrors.WithMessagef(err, "time.ParseDuration(Connect_Timeout{%#v})", consumerConfig.Connect_Timeout) } } - return err + + return nil } diff --git a/config/generic_service.go b/config/generic_service.go index 9895486e97..b66e399f9e 100644 --- a/config/generic_service.go +++ b/config/generic_service.go @@ -17,9 +17,11 @@ package config +import "context" + // GenericService ... type GenericService struct { - Invoke func(req []interface{}) (interface{}, error) `dubbo:"$invoke"` + Invoke func(ctx context.Context, req []interface{}) (interface{}, error) `dubbo:"$invoke"` referenceStr string } diff --git a/config/interfaces/config_reader.go b/config/interfaces/config_reader.go new file mode 100644 index 0000000000..23f2225e1b --- /dev/null +++ b/config/interfaces/config_reader.go @@ -0,0 +1,9 @@ +package interfaces + +import "bytes" + +// ConfigReader +type ConfigReader interface { + ReadConsumerConfig(reader *bytes.Buffer) error + ReadProviderConfig(reader *bytes.Buffer) error +} diff --git a/config/provider_config.go b/config/provider_config.go index a36fd4d0a0..14b77cafb3 100644 --- a/config/provider_config.go +++ b/config/provider_config.go @@ -17,6 +17,10 @@ package config +import ( + "bytes" +) + import ( "github.com/creasty/defaults" perrors "github.com/pkg/errors" @@ -25,6 +29,7 @@ import ( import ( "github.com/apache/dubbo-go/common/constant" "github.com/apache/dubbo-go/common/logger" + "github.com/apache/dubbo-go/common/yaml" ) ///////////////////////// @@ -45,6 +50,7 @@ type ProviderConfig struct { ProtocolConf interface{} `yaml:"protocol_conf" json:"protocol_conf,omitempty" property:"protocol_conf" ` FilterConf interface{} `yaml:"filter_conf" json:"filter_conf,omitempty" property:"filter_conf" ` ShutdownConfig *ShutdownConfig `yaml:"shutdown_conf" json:"shutdown_conf,omitempty" property:"shutdown_conf" ` + ConfigType map[string]string `yaml:"config_type" json:"config_type,omitempty" property:"config_type"` } // UnmarshalYAML ... @@ -74,13 +80,13 @@ func ProviderInit(confProFile string) error { if len(confProFile) == 0 { return perrors.Errorf("application configure(provider) file name is nil") } - providerConfig = &ProviderConfig{} - err := unmarshalYMLConfig(confProFile, providerConfig) + fileStream, err := yaml.UnmarshalYMLConfig(confProFile, providerConfig) if err != nil { - return perrors.Errorf("yaml.Unmarshal() = error:%v", perrors.WithStack(err)) + return perrors.Errorf("unmarshalYmlConfig error %v", perrors.WithStack(err)) } + providerConfig.fileStream = bytes.NewBuffer(fileStream) //set method interfaceId & interfaceName for k, v := range providerConfig.Services { //set id for reference diff --git a/config_center/nacos/client.go b/config_center/nacos/client.go index 1bf61a942b..d3373e249b 100644 --- a/config_center/nacos/client.go +++ b/config_center/nacos/client.go @@ -157,7 +157,7 @@ func newNacosClient(name string, nacosAddrs []string, timeout time.Duration) (*N }, } - svrConfList := []nacosconst.ServerConfig{} + svrConfList := make([]nacosconst.ServerConfig, 0, len(n.NacosAddrs)) for _, nacosAddr := range n.NacosAddrs { split := strings.Split(nacosAddr, ":") port, err := strconv.ParseUint(split[1], 10, 64) diff --git a/config_center/nacos/facade.go b/config_center/nacos/facade.go index fc83e14eac..77a79ed091 100644 --- a/config_center/nacos/facade.go +++ b/config_center/nacos/facade.go @@ -46,15 +46,10 @@ type nacosClientFacade interface { common.Node } -func timeSecondDuration(sec int) time.Duration { - return time.Duration(sec) * time.Second -} - // HandleClientRestart Restart client handler func HandleClientRestart(r nacosClientFacade) { var ( - err error - + err error failTimes int ) @@ -79,7 +74,7 @@ LOOP: case <-r.GetDone(): logger.Warnf("(NacosProviderRegistry)reconnectZkRegistry goroutine exit now...") break LOOP - case <-getty.GetTimeWheel().After(timeSecondDuration(failTimes * connDelay)): // Prevent crazy reconnection nacos. + case <-getty.GetTimeWheel().After(time.Duration(failTimes*connDelay) * time.Second): // Prevent crazy reconnection nacos. } err = ValidateNacosClient(r, WithNacosName(nacosName)) logger.Infof("NacosProviderRegistry.validateNacosClient(nacosAddr{%s}) = error{%#v}", diff --git a/config_center/parser/configuration_parser.go b/config_center/parser/configuration_parser.go index f342dc62e7..f33b4ba866 100644 --- a/config_center/parser/configuration_parser.go +++ b/config_center/parser/configuration_parser.go @@ -233,18 +233,10 @@ func getParamString(item ConfigItem) (string, error) { "you want to change in the rule.") } for k, v := range params { - retStr = retStr + "&" - retStr = retStr + k - retStr = retStr + "=" - retStr = retStr + v + retStr += "&" + k + "=" + v } - if len(item.ProviderAddresses) >= 0 { - retStr = retStr + "&" - retStr = retStr + constant.OVERRIDE_PROVIDERS_KEY - retStr = retStr + "=" - retStr = retStr + strings.Join(item.ProviderAddresses, ",") - } + retStr += "&" + constant.OVERRIDE_PROVIDERS_KEY + "=" + strings.Join(item.ProviderAddresses, ",") return retStr, nil } diff --git a/doc/apache/apache-release-procedure-20200306.md b/doc/apache/apache-release-procedure-20200306.md new file mode 100644 index 0000000000..3f677ff56b --- /dev/null +++ b/doc/apache/apache-release-procedure-20200306.md @@ -0,0 +1,448 @@ + +# Apache 软件发版流程 + +> author: wongoo@apache.org +> last updated: 2020-03-06 + +Apache开源软件是有社区驱动的，为了提高发布软件质量而指定了软件发布流程，本文主要介绍此流程，以给第一次发布打包的apacher参考。 + +如果你要准备打包一个apache软件了，想必你已经是一个项目的committer了，而且知道社区、PMC这些概念，而你现在还担任本次发布的 release manager 一职。 + +发版流程其实也很简单，无非如下： +1. 整理变更内容，打包并对打包文件签名； +2. 将签名文件上传apache svn仓库； +3. 发邮件请社区PMC大佬投票； +4. 投票通过后发一个投票结果通告邮件； +5. 发版 +6. 发版邮件通告社区新版本发布； + +下面详细整理发版的一些流程步骤，使用 dubbo 的子项目 dubbog-go-hessian2 发版为例！ + + +## 1. 发版准备 + +发版文件需要签名，需要安装pgp工具. + +```bash +$ brew install gpg +$ gpg --version +$ gpg --full-gen-key + (1) RSA and RSA (default) <-- RSA 类型 + What keysize do you want? (2048) 4096 <-- key大小为4096 + 0 = key does not expire <-- 永不过期 + Real name: Liu Yang + Email address: wongoo@apache.org + Comment: CODE SIGNING KEY + + gpg: /Users/gelnyang/.gnupg/trustdb.gpg: trustdb created + gpg: key 7DB68550D366E4C0 marked as ultimately trusted + gpg: revocation certificate stored as '/Users/gelnyang/.gnupg/openpgp-revocs.d/1376A2FF67E4C477573909BD7DB68550D366E4C0.rev' + public and secret key created and signed. + + pub rsa4096 2019-10-17 [SC] + 1376A2FF67E4C477573909BD7DB68550D366E4C0 + uid Liu Yang (CODE SIGNING KEY) + sub rsa4096 2019-10-17 [E] + +$ gpg --list-keys + gpg: checking the trustdb + gpg: marginals needed: 3 completes needed: 1 trust model: pgp + gpg: depth: 0 valid: 1 signed: 0 trust: 0-, 0q, 0n, 0m, 0f, 1u + /Users/gelnyang/.gnupg/pubring.kbx + ---------------------------------- + pub rsa4096 2019-10-17 [SC] + 1376A2FF67E4C477573909BD7DB68550D366E4C0 + uid [ultimate] Liu Yang (CODE SIGNING KEY) + sub rsa4096 2019-10-17 [E] + + +# 公钥服务器是网络上专门储存用户公钥的服务器 +# 通过key id发送public key到keyserver +$ gpg --keyserver pgpkeys.mit.edu --send-key 1376A2FF67E4C477573909BD7DB68550D366E4C0 + gpg: sending key 7DB68550D366E4C0 to hkp://pgpkeys.mit.edu +# 其中，pgpkeys.mit.edu为随意挑选的keyserver，keyserver列表为：https://sks-keyservers.net/status/，为相互之间是自动同步的，选任意一个都可以。 + +# 如果有多个public key，设置默认key。修改 ~/.gnupg/gpg.conf +$ vi ~/.gnupg/gpg.conf +default-key 7DB68550D366E4C0 + +# 如果有多个public key, 也可以删除无用的key： +### 先删除私钥，再删除公钥 +$ gpg --yes --delete-secret-keys shenglicao2@gmail.com ###老的私钥，指明邮箱即可 +$ gpg --delete-keys 1808C6444C781C0AEA0AAD4C4D6A8007D20DB8A4 + +## 由于公钥服务器没有检查机制，任何人都可以用你的名义上传公钥，所以没有办法保证服务器上的公钥的可靠性。 +## 通常，你可以在网站上公布一个公钥指纹，让其他人核对下载到的公钥是否为真。 +# fingerprint参数生成公钥指纹： +$ gpg --fingerprint wongoo + + pub rsa4096 2019-10-17 [SC] + 1376 A2FF 67E4 C477 5739 09BD 7DB6 8550 D366 E4C0 + uid [ultimate] Liu Yang (CODE SIGNING KEY) + sub rsa4096 2019-10-17 [E] + # 将上面的 fingerprint （即 1376 A2FF 67E4 C477 5739 09BD 7DB6 8550 D366 E4C0）粘贴到自己的用户信息中： + # https://id.apache.org OpenPGP Public Key Primary Fingerprint: +``` + +> 详细参考： +> - 发布签名: http://www.apache.org/dev/release-signing.html +> - 发布策略: http://www.apache.org/dev/release-distribution +> - 将密钥上传到公共密钥服务器: https://www.apache.org/dev/openpgp.html#generate-key + +## 2. 打包签名 + +准备打包前（尤其提第一次打包）需要注意以下内容: +- 每个文件的LICENSE头部是否正确, 包括 `*.java`, `*.go`, `*.xml`, `Makefile` 等 +- LICENSE 文件是否存在 +- NOTICE 文件是否存在 +- CHANGE.md 是否存在 （变更内容格式符合规范） + +以上可以参考其他已发布项目的配置。 + + +``` + +# NOTICE: 这里切分支，分支名称不要和版本号（tag用）类似，不然会有冲突 +$ git checkout -b 1.4 + +$ git tag -a v1.4.0-rc1 -m "v1.4.0 release candidate 1" + +$ git push --tags + +# 打包 +$ git archive --format=tar 1.4 --prefix=dubbo-go-hessian2-v1.4.0/ | gzip > dubbo-go-hessian2-v1.4.0-src.tar.gz + +# 签名 +$ gpg -u wongoo@apache.org --armor --output dubbo-go-hessian2-v1.4.0-src.tar.gz.asc --detach-sign dubbo-go-hessian2-v1.4.0-src.tar.gz + +# 验证签名 +$ gpg --verify dubbo-go-hessian2-v1.4.0-src.tar.gz.asc dubbo-go-hessian2-v1.4.0-src.tar.gz + +# hash +$ shasum -a 512 dubbo-go-hessian2-v1.4.0-src.tar.gz > dubbo-go-hessian2-v1.4.0-src.tar.gz.sha512 + +# 验证 hash +$ shasum --check dubbo-go-hessian2-v1.4.0-src.tar.gz.sha512 + +``` + +> 发布版本: http://www.apache.org/dev/release-publishing.html + +## 3. 上传打包文件到svn仓库 + +``` +$ svn checkout https://dist.apache.org/repos/dist/dev/dubbo + +$ cd dubbo + +# 更新 +$ svn update + +# 添加 签名 和 public key 到KEYS文件并提交到SVN仓库 +# 这里是将公钥KEYS放到根目录, 有的项目放到本次打包文件目录 +$ (gpg --list-sigs wongoo && gpg --armor --export wongoo) >> KEYS + +$ mkdir -p dubbo-go-hessian2/v1.4.0-rc1 + +# 拷贝相关文件到新建目录下 + +$ tree dubbo-go-hessian2 +dubbo-go-hessian2 +└── v1.4.0-rc1 + ├── dubbo-go-hessian2-v1.4.0-src.tar.gz + ├── dubbo-go-hessian2-v1.4.0-src.tar.gz.asc + └── dubbo-go-hessian2-v1.4.0-src.tar.gz.sha512 + +$ svn add dubbo-go-hessian2 +$ svn add dubbo-go-hessian2/* +$ svn status +$ svn commit --username wongoo -m "Release dubbo-go-hessian2 v1.4.0-rc1" +``` + +> 详细参考: svn版本管理 https://www.apache.org/dev/version-control.html + + +## 4. 发投票 [VOTE] 邮件 + +发任何邮件都是有一定格式的，你加入社区邮件列表后，就会收到很多这样的邮件，多看看就知道了，具体邮件范本参考文章后面的邮件范本。 + +发完【VOTE】邮件，私下沟通群里面请大佬PMC投票。 +PMC投票会对你上传打包文件进行相关检查, +详细可以了解孵化中的项目发布完整的检查项参考： https://cwiki.apache.org/confluence/display/INCUBATOR2/IncubatorReleaseChecklist + +收到3个binding邮件且超过72小时后，就可以发 投票结果 [RESULT] [VOTE] 邮件了。 + +> 原则上只有PMC的投票才算binding邮件, 当然也可以由社区决定。 + +这一步骤最常见有以下问题： +- 文件签名有问题 +- 引用项目LICENSE问题 +- 单元测试不通过 + +> 另外需要注意: 一个apache项目可能包含很多子项目，项目的PMC可能只对主项目比较了解， 他们并不清楚如何将子项目跑起来，也不知道如何跑单元测试，最好在邮件中附带一个如何进行单元测试的连接。例如 PMC 最了解 java，但子项目是golang，python，js等，你需要告诉他们如何测试你的项目。 + +可以参考投票规则: https://www.apache.org/foundation/voting.html + +## 5. 发布版本 + +当正式发布投票成功后，先发[Result]邮件，然后就准备 release package。 +将之前在dev下发布的对应rc文件夹下的源码包、签名文件和hash文件拷贝到另一个目录 v1.4.0， +注意文件名字中不要rcxx (可以rename，但不要重新计算签名，hash可以重新计算，结果不会变)。 + +将release包移动到正式版目录。如果你的软件是需要客户从apache下载的，则这一步是必须的。如果不是，比如golang引用github打包地址的则可以忽略。 +``` +svn up +cd dubbo-go-hessian2 +svn move v1.4.0-rc1 v1.4.0 +svn status +svn commit --username wongoo -m "Release dubbo-go-hessian2 v1.4.0" +``` + +移到发版目录后，还需要进行相应的正式版本发布， 这里将具体发布方式整理到单独的章节 `7. 不同语言版本发布`，因为发布流程马上就要结束了 ^v^ + + +## 6. 新版本通告 ANNOUNCE 邮件 + +恭喜你你已经到发版最后一步了，邮件格式参考以下邮件范本！ + + +## 7. 不同语言版本发布 + +### 7.1 golang + +在 github 基于投票分支发布了 release 版本。 + +### 7.2 java + +java项目发版需发布到java maven仓库。 + +TODO + +### 7.3 js + +js项目发版需发布到npm仓库。 + +TODO + +### 7.4 python + +TODO + +## 8. 邮件范本 + +### 8.1. 提出发版投票 + +- TO: dev@dubbo.apache.org +- Title: [VOTE] Release Apache dubbo-go-hessian2 v1.4.0 RC1 + +``` +Hello Dubbo/Dubbogo Community, + + This is a call for vote to release Apache dubbo-go-hessian2 version v1.4.0 RC1. + + The release candidates: https://dist.apache.org/repos/dist/dev/dubbo/dubbo-go-hessian2/v1.4.0-rc1/ + Git tag for the release: https://github.com/apache/dubbo-go-hessian2/tree/1.4 + Hash for the release tag: 4c31e88c35afe84c0321d9f12f036e6d3c8962d0 + Release Notes: https://github.com/apache/dubbo-go-hessian2/blob/1.4/CHANGE.md + The artifacts have been signed with Key :7DB68550D366E4C0, which can be found in the keys file: + https://dist.apache.org/repos/dist/dev/dubbo/KEYS + + The vote will be open for at least 72 hours or until necessary number of votes are reached. + + Please vote accordingly: + [ ] +1 approve + [ ] +0 no opinion + [ ] -1 disapprove with the reason + + Thanks, + The Apache Dubbo-go Team + ``` + + +### 8.2. PMC 投票邮件回复 + + +范例1： +``` ++1 approve <-- 首先表明同不同意 + +I have checked: <-- 其次要说明自己检查了哪些项 + +1.source code can build <-- 能否构建 +2.tests can pass in my local <-- 单元测试能否通过 +3. NOTICE LICENSE file exist <-- 协议文件是否存在 +4.git tag is correct <-- git tag 是否正确 + +there is one minor thing that in change logs file, there is no space +between text And link. I suggest add one to make it looks better. <--- 一些其他改进建议 +``` + +范例2： +``` ++1 + +I checked the following items: + +[v] Are release files in correct location? <-- 发布文件目录是否正确 +[v] Do release files have the word incubating in their name? +[v] Are the digital signature and hashes correct? <-- 签名、hash是否正确 +[v] Do LICENSE and NOTICE files exists? +[v] Is the LICENSE and NOTICE text correct? <-- 协议文本是否正确 +[v] Is the NOTICE year correct? <-- 注意年份是否正确 +[v] Un-included software dependencies are not mentioned in LICENSE or NOTICE? <-- 没有包含协议或注意没有提到的软件依赖 +[v] License information is not mentioned in NOTICE? <-- 协议信息没有在注意中提及 +[x] Is there any 3rd party code contained inside the release? If so: <-- 是否包含第三方代码 + [ ] Does the software have a compatible license? + [ ] Are all software licenses mentioned in LICENSE? + [ ] Is the full text of the licenses (or pointers to it) in LICENSE? + Is any of this code Apache licensed? Do they have NOTICE files? If so: + [ ] Have relevant parts of those NOTICE files been added to this NOTICE file? +[v] Do all source files have ASF headers? <-- 是否所有源码都有ASF头部 +[v] Do the contents of the release match with what's tagged in version control? <-- 发布的文件是否和github中tag标记的版本一致 +[x] Are there any unexpected binary files in the release? <-- 是否包含不应该存在的二进制文件 +[v] Can you compile from source? Are the instruction clear? <-- 能否编译？指令是否明确？ + +On my mac laptop, I could compile successfully but there's one failed unit +test against net.go. I believe this issue [1] can be fixed with [2] in the +next release. <-- 编译问题及建议 + +Is the issue minor? <-- 编译存在的问题是否都是较小的？ +[v] Yes [ ] No [ ] Unsure + +Could it possibly be fixed in the next release? <-- 能否在下一版本修复？ +[v] Yes [ ] No [ ] Unsure + +I vote with: <-- 我的投票 +[v] +1 release the software +[ ] +0 not sure if it should be released +[ ] -1 don’t release the software because... + +Regards, +-Ian. + +1. https://github.com/apache/dubbo-go/issues/207 +2. https://github.com/apache/dubbo-go/pull/209 +``` + +范例3： +``` ++1 + +I checked the following items: + +[√] Do LICENSE and NOTICE files exists? +[√] Is the LICENSE and NOTICE text correct? +[√] Is the NOTICE year correct? +[√] Do all source files have ASF headers? +[√] Do the contents of the release match with what's tagged in version control? +[√] Can you compile from source? +I could compile successfully but there's failed units test. I run the unit +test refer to :https://github.com/apache/dubbo-go#running-unit-tests . +But I think it is not matter, the test can be fixed in next release. + + +I vote with: +[√] +1 release the software +``` + +范例4: +``` +Great improvement over the previous release but there are still issues from the last vote that have not been resolved. e.g. [6][7][8] + +Can someone tell me if these files [1][2][3][4][5] are just missing ASF headers or have a different license? + +If they are just missing headers and [6][7][8] explained then it +1 form me, otherwise it’s probably a -1. + +Can people please carefully check the contents, and write down what you checked, rather than just saying +1. + +I checked: +- signatures and hashes good +- LICENSE is missing the appendix (not a major issue) +- LICENSE may be is missing some information[1][2][3][4][5] +- NOTICE is fine +- No binaries in source release +- Some files are missing ASF headers or other license headers [1][2][3][4][5] - please fix + +Thanks, +Justin + +1. dubbo-go-1.1.0/cluster/loadbalance/round_robin_test.go +2. dubbo-go-1.1.0/common/extension/router_factory.go +3. dubbo-go-1.1.0/config_center/configuration_parser.go +4. dubbo-go-1.1.0/config_center/configuration_parser_test.go +5. dubbo-go-1.1.0/registry/zookeeper/listener_test.go +6. dubbo-go-1.1.0/cluster/loadbalance/least_active.go +7. dubbo-go-1.1.0/protocol/RpcStatus.go +8. dubbo-go-1.1.0/filter/impl/active_filter.go +``` + + +### 8.3. 发 [RESULT] [VOTE] 投票结果通知邮件 + +- TO: dev@dubbo.apache.org +- Title: [RESULT] [VOTE]: Release Apache dubbo-go-hessian2 v1.4.0 RC1 + + +``` +Hello Dubbo/Dubbogo Community, + +The release dubbo-go-hessian2 v1.4.0 RC1 vote finished, We’ve received 3 +1 (binding) votes. + ++1 binding, Stocks Alex ++1 binding, Ian Luo ++1 binding, Jun Liu + +The vote and result thread: +https://lists.apache.org/thread.html/r8070f3b00984888069dd4ddad1bbc424cde51ea68b6ff0520e609e18%40%3Cdev.dubbo.apache.org%3E + + +The vote passed. Thanks all. +I will proceed with the formal release later. + + +Best regards, + +The Apache Dubbogo Team +``` + + +### 8.4. 发 Announce 发版邮件 + +- TO: dev@dubbo.apache.org +- [ANNOUNCE] Apache Dubbo version 2.7.4 Released + +``` +Hello Community, + +The Apache Dubbo team is pleased to announce that the 2.7.4 has been +released. + +Apache Dubbo™ is a high-performance, java based, open source +RPC framework. Dubbo offers three key functionalities, which include +interface based remote call, fault tolerance & load balancing, and +automatic service registration & discovery. + +Both the source release[1] and the maven binary release[2] are available +now, you can also find the detailed release notes in here[3]. + + +If you have any usage questions, or have problems when upgrading or find +any problems about enhancements included in this release, please don’t +hesitate to let us know by sending feedback to this mailing list or filing +an issue on GitHub[4]. + + + +[1] http://dubbo.apache.org/en-us/blog/download.html +[2] http://central.maven.org/maven2/org/apache/dubbo +[3] https://github.com/apache/dubbo/releases +[4] https://github.com/apache/dubbo/issues +``` + +## 9. 参考 + +- dubbo发布流程: http://dubbo.apache.org/zh-cn/docs/developers/committer-guide/release-guide_dev.html +- doris发布流程: https://github.com/apache/incubator-doris/blob/master/docs/documentation/cn/community/release-process.md +- spark发布流程: http://spark0apache0org.icopy.site/release-process.html + + diff --git a/doc/apache/release_note.md b/doc/apache/release_note.md new file mode 100644 index 0000000000..747a3348a1 --- /dev/null +++ b/doc/apache/release_note.md @@ -0,0 +1,11 @@ +### How to release a new version? +--- + +* 1 Check the time range of NOTICE; +* 2 Add the features to the feature list of README.md/README_CN.md/CHANGE.md; +* 3 Check whether every code file has the Apache License 2.0 or not; +* 4 There should not be author info(name & email etc) exist in code file; +* 5 Run all unit tests; +* 6 Run all samples in apache/dubbo-samples/golang; +* 7 Write "What's New" by releaser who should be an apache/dubbo-go committer; +* 8 And then, u can release a new version refer to [Apache 软件发版流程](./apache-release-procedure-20200306.md); \ No newline at end of file diff --git a/go.mod b/go.mod index b977df68c3..83091cf8b9 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ require ( github.com/Workiva/go-datastructures v1.0.50 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190802083043-4cd0c391755e // indirect - github.com/apache/dubbo-go-hessian2 v1.3.1-0.20200302092433-6ae5479d93a3 + github.com/apache/dubbo-go-hessian2 v1.4.0 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23 // indirect github.com/coreos/bbolt v1.3.3 // indirect github.com/coreos/etcd v3.3.13+incompatible @@ -15,8 +15,10 @@ require ( github.com/dubbogo/getty v1.3.3 github.com/dubbogo/go-zookeeper v1.0.0 github.com/dubbogo/gost v1.5.2 + github.com/emicklei/go-restful/v3 v3.0.0 github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 // indirect github.com/go-errors/errors v1.0.1 // indirect + github.com/go-resty/resty/v2 v2.1.0 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/golang/mock v1.3.1 github.com/golang/protobuf v1.3.2 @@ -24,6 +26,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.9.5 // indirect + github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 github.com/hashicorp/consul v1.5.3 github.com/hashicorp/consul/api v1.1.0 github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 // indirect diff --git a/go.sum b/go.sum index c2be2b1e43..813496b6ee 100644 --- a/go.sum +++ b/go.sum @@ -38,8 +38,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190802083043-4cd0c391755e h1:MSuLXx/mveDbpDNhVrcWTMeV4lbYWKcyO4rH+jAxmX0= github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190802083043-4cd0c391755e/go.mod h1:myCDvQSzCW+wB1WAlocEru4wMGJxy+vlxHdhegi1CDQ= github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/apache/dubbo-go-hessian2 v1.3.1-0.20200302092433-6ae5479d93a3 h1:1HM47ILUkLaMxLKUub+WHPncqrJGEQ0KRJzSJueMDpY= -github.com/apache/dubbo-go-hessian2 v1.3.1-0.20200302092433-6ae5479d93a3/go.mod h1:VwEnsOMidkM1usya2uPfGpSLO9XUF//WQcWn3y+jFz8= +github.com/apache/dubbo-go-hessian2 v1.4.0 h1:Cb9FQVTy3G93dnDr7P93U8DeKFYpDTJjQp44JG5TafA= +github.com/apache/dubbo-go-hessian2 v1.4.0/go.mod h1:VwEnsOMidkM1usya2uPfGpSLO9XUF//WQcWn3y+jFz8= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -110,7 +110,6 @@ github.com/dubbogo/getty v1.3.3 h1:8m4zZBqFHO+NmhH7rMPlFuuYRVjcPD7cUhumevqMZZs= github.com/dubbogo/getty v1.3.3/go.mod h1:U92BDyJ6sW9Jpohr2Vlz8w2uUbIbNZ3d+6rJvFTSPp0= github.com/dubbogo/go-zookeeper v1.0.0 h1:RsYdlGwhDW+iKXM3eIIcvt34P2swLdmQfuIJxsHlGoM= github.com/dubbogo/go-zookeeper v1.0.0/go.mod h1:fn6n2CAEer3novYgk9ULLwAjuV8/g4DdC2ENwRb6E+c= -github.com/dubbogo/gost v1.5.1 h1:oG5dzaWf1KYynBaBoUIOkgT+YD0niHV6xxI0Odq7hDg= github.com/dubbogo/gost v1.5.1/go.mod h1:pPTjVyoJan3aPxBPNUX0ADkXjPibLo+/Ib0/fADXSG8= github.com/dubbogo/gost v1.5.2 h1:ri/03971hdpnn3QeCU+4UZgnRNGDXLDGDucR/iozZm8= github.com/dubbogo/gost v1.5.2/go.mod h1:pPTjVyoJan3aPxBPNUX0ADkXjPibLo+/Ib0/fADXSG8= @@ -118,7 +117,10 @@ github.com/duosecurity/duo_api_golang v0.0.0-20190308151101-6c680f768e74 h1:2MIh github.com/duosecurity/duo_api_golang v0.0.0-20190308151101-6c680f768e74/go.mod h1:UqXY1lYT/ERa4OEAywUqdok1T4RCRdArkhic1Opuavo= github.com/elazarl/go-bindata-assetfs v0.0.0-20160803192304-e1a2a7ec64b0 h1:ZoRgc53qJCfSLimXqJDrmBhnt5GChDsExMCK7t48o0Y= github.com/elazarl/go-bindata-assetfs v0.0.0-20160803192304-e1a2a7ec64b0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.0.0 h1:Duxxa4x0WIHW3bYEDmoAPNjmy8Rbqn+utcF74dlF/G8= +github.com/emicklei/go-restful/v3 v3.0.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.8.0 h1:uE6Fp4fOcAJdc1wTQXLJ+SYistkbG1dNoi6Zs1+Ybvk= github.com/envoyproxy/go-control-plane v0.8.0/go.mod h1:GSSbY9P1neVhdY7G4wu+IK1rk/dqhiCC/4ExuWJZVuk= github.com/envoyproxy/protoc-gen-validate v0.0.14 h1:YBW6/cKy9prEGRYLnaGa4IDhzxZhRCtKsax8srGKDnM= @@ -149,6 +151,8 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+ github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-resty/resty/v2 v2.1.0 h1:Z6IefCpUMfnvItVJaJXWv/pMiiD11So35QgwEELsldE= +github.com/go-resty/resty/v2 v2.1.0/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-sql-driver/mysql v0.0.0-20180618115901-749ddf1598b4 h1:1LlmVz15APoKz9dnm5j2ePptburJlwEH+/v/pUuoxck= github.com/go-sql-driver/mysql v0.0.0-20180618115901-749ddf1598b4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -212,6 +216,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/consul v1.5.3 h1:EmTWRf/cuqZk6Ug9tgFUVE9xNgJPpmBvJwJMvm+agSk= @@ -453,7 +459,6 @@ github.com/smartystreets/assertions v0.0.0-20180820201707-7c9eb446e3cf/go.mod h1 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 h1:N8Bg45zpk/UcpNGnfJt2y/3lRWASHNTUET8owPYCgYI= github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -470,9 +475,7 @@ github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -505,7 +508,6 @@ go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -520,8 +522,9 @@ golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20170807180024-9a379c6b3e95/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -544,7 +547,6 @@ golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -558,7 +560,6 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= google.golang.org/api v0.0.0-20180829000535-087779f1d2c9 h1:z1TeLUmxf9ws9KLICfmX+KGXTs+rjm+aGWzfsv7MZ9w= google.golang.org/api v0.0.0-20180829000535-087779f1d2c9/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= @@ -571,7 +572,6 @@ google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.22.1 h1:/7cs52RnTJmD43s3uxzlq2U7nqVTd/37viQwMrMNlOM= google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= diff --git a/protocol/grpc/client.go b/protocol/grpc/client.go index d35a2c770c..6026f0991b 100644 --- a/protocol/grpc/client.go +++ b/protocol/grpc/client.go @@ -22,6 +22,8 @@ import ( ) import ( + "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" + "github.com/opentracing/opentracing-go" "google.golang.org/grpc" ) @@ -39,7 +41,11 @@ type Client struct { // NewClient ... func NewClient(url common.URL) *Client { - conn, err := grpc.Dial(url.Location, grpc.WithInsecure(), grpc.WithBlock()) + // if global trace instance was set , it means trace function enabled. If not , will return Nooptracer + tracer := opentracing.GlobalTracer() + conn, err := grpc.Dial(url.Location, grpc.WithInsecure(), grpc.WithBlock(), + grpc.WithUnaryInterceptor( + otgrpc.OpenTracingClientInterceptor(tracer, otgrpc.LogPayloads()))) if err != nil { panic(err) } diff --git a/protocol/grpc/server.go b/protocol/grpc/server.go index 19b9db4ac7..cc184bf3cf 100644 --- a/protocol/grpc/server.go +++ b/protocol/grpc/server.go @@ -24,6 +24,8 @@ import ( ) import ( + "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" + "github.com/opentracing/opentracing-go" "google.golang.org/grpc" ) @@ -63,7 +65,11 @@ func (s *Server) Start(url common.URL) { if err != nil { panic(err) } - server := grpc.NewServer() + + // if global trace instance was set , then server tracer instance can be get. If not , will return Nooptracer + tracer := opentracing.GlobalTracer() + server := grpc.NewServer( + grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(tracer))) key := url.GetParam(constant.BEAN_NAME_KEY, "") service := config.GetProviderService(key) diff --git a/protocol/rest/client/client_impl/resty_client.go b/protocol/rest/client/client_impl/resty_client.go new file mode 100644 index 0000000000..aa6c23137d --- /dev/null +++ b/protocol/rest/client/client_impl/resty_client.go @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client_impl + +import ( + "context" + "net" + "net/http" + "path" + "time" +) + +import ( + "github.com/go-resty/resty/v2" + perrors "github.com/pkg/errors" +) + +import ( + "github.com/apache/dubbo-go/common/constant" + "github.com/apache/dubbo-go/common/extension" + "github.com/apache/dubbo-go/protocol/rest/client" +) + +func init() { + extension.SetRestClient(constant.DEFAULT_REST_CLIENT, NewRestyClient) +} + +type RestyClient struct { + client *resty.Client +} + +func NewRestyClient(restOption *client.RestOptions) client.RestClient { + client := resty.New() + client.SetTransport( + &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + c, err := net.DialTimeout(network, addr, restOption.ConnectTimeout) + if err != nil { + return nil, err + } + err = c.SetDeadline(time.Now().Add(restOption.RequestTimeout)) + if err != nil { + return nil, err + } + return c, nil + }, + }) + return &RestyClient{ + client: client, + } +} + +func (rc *RestyClient) Do(restRequest *client.RestRequest, res interface{}) error { + r, err := rc.client.R(). + SetHeader("Content-Type", restRequest.Consumes). + SetHeader("Accept", restRequest.Produces). + SetPathParams(restRequest.PathParams). + SetQueryParams(restRequest.QueryParams). + SetHeaders(restRequest.Headers). + SetBody(restRequest.Body). + SetResult(res). + Execute(restRequest.Method, "http://"+path.Join(restRequest.Location, restRequest.Path)) + if err != nil { + return perrors.WithStack(err) + } + if r.IsError() { + return perrors.New(r.String()) + } + return nil +} diff --git a/protocol/rest/client/rest_client.go b/protocol/rest/client/rest_client.go new file mode 100644 index 0000000000..7d020abc81 --- /dev/null +++ b/protocol/rest/client/rest_client.go @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "time" +) + +type RestOptions struct { + RequestTimeout time.Duration + ConnectTimeout time.Duration +} + +type RestRequest struct { + Location string + Path string + Produces string + Consumes string + Method string + PathParams map[string]string + QueryParams map[string]string + Body interface{} + Headers map[string]string +} + +type RestClient interface { + Do(request *RestRequest, res interface{}) error +} diff --git a/protocol/rest/config/reader/rest_config_reader.go b/protocol/rest/config/reader/rest_config_reader.go new file mode 100644 index 0000000000..873af9924b --- /dev/null +++ b/protocol/rest/config/reader/rest_config_reader.go @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package reader + +import ( + "bytes" + "strconv" + "strings" +) + +import ( + perrors "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +import ( + "github.com/apache/dubbo-go/common/constant" + "github.com/apache/dubbo-go/common/extension" + "github.com/apache/dubbo-go/common/logger" + "github.com/apache/dubbo-go/config/interfaces" + "github.com/apache/dubbo-go/protocol/rest/config" +) + +const REST = "rest" + +func init() { + extension.SetConfigReaders(REST, NewRestConfigReader) + extension.SetDefaultConfigReader(REST, REST) +} + +type RestConfigReader struct { +} + +func NewRestConfigReader() interfaces.ConfigReader { + return &RestConfigReader{} +} + +// ReadConsumerConfig read consumer config for rest protocol +func (cr *RestConfigReader) ReadConsumerConfig(reader *bytes.Buffer) error { + restConsumerConfig := &config.RestConsumerConfig{} + err := yaml.Unmarshal(reader.Bytes(), restConsumerConfig) + if err != nil { + return perrors.Errorf("[Rest Config] unmarshal Consumer error %#v", perrors.WithStack(err)) + } + + restConsumerServiceConfigMap := make(map[string]*config.RestServiceConfig, len(restConsumerConfig.RestServiceConfigsMap)) + for key, rc := range restConsumerConfig.RestServiceConfigsMap { + rc.Client = getNotEmptyStr(rc.Client, restConsumerConfig.Client, constant.DEFAULT_REST_CLIENT) + rc.RestMethodConfigsMap = initMethodConfigMap(rc, restConsumerConfig.Consumes, restConsumerConfig.Produces) + restConsumerServiceConfigMap[strings.TrimPrefix(key, "/")] = rc + } + config.SetRestConsumerServiceConfigMap(restConsumerServiceConfigMap) + return nil +} + +// ReadProviderConfig read provider config for rest protocol +func (cr *RestConfigReader) ReadProviderConfig(reader *bytes.Buffer) error { + restProviderConfig := &config.RestProviderConfig{} + err := yaml.Unmarshal(reader.Bytes(), restProviderConfig) + if err != nil { + return perrors.Errorf("[Rest Config] unmarshal Provider error %#v", perrors.WithStack(err)) + } + restProviderServiceConfigMap := make(map[string]*config.RestServiceConfig, len(restProviderConfig.RestServiceConfigsMap)) + for key, rc := range restProviderConfig.RestServiceConfigsMap { + rc.Server = getNotEmptyStr(rc.Server, restProviderConfig.Server, constant.DEFAULT_REST_SERVER) + rc.RestMethodConfigsMap = initMethodConfigMap(rc, restProviderConfig.Consumes, restProviderConfig.Produces) + restProviderServiceConfigMap[strings.TrimPrefix(key, "/")] = rc + } + config.SetRestProviderServiceConfigMap(restProviderServiceConfigMap) + return nil +} + +// initProviderRestConfig ... +func initMethodConfigMap(rc *config.RestServiceConfig, consumes string, produces string) map[string]*config.RestMethodConfig { + mcm := make(map[string]*config.RestMethodConfig, len(rc.RestMethodConfigs)) + for _, mc := range rc.RestMethodConfigs { + mc.InterfaceName = rc.InterfaceName + mc.Path = rc.Path + mc.Path + mc.Consumes = getNotEmptyStr(mc.Consumes, rc.Consumes, consumes) + mc.Produces = getNotEmptyStr(mc.Produces, rc.Produces, produces) + mc.MethodType = getNotEmptyStr(mc.MethodType, rc.MethodType) + mc = transformMethodConfig(mc) + mcm[mc.MethodName] = mc + } + return mcm +} + +// function will return first not empty string .. +func getNotEmptyStr(args ...string) string { + var r string + for _, t := range args { + if len(t) > 0 { + r = t + break + } + } + return r +} + +// transformMethodConfig +func transformMethodConfig(methodConfig *config.RestMethodConfig) *config.RestMethodConfig { + if len(methodConfig.PathParamsMap) == 0 && len(methodConfig.PathParams) > 0 { + paramsMap, err := parseParamsString2Map(methodConfig.PathParams) + if err != nil { + logger.Warnf("[Rest Config] Path Param parse error:%v", err) + } else { + methodConfig.PathParamsMap = paramsMap + } + } + if len(methodConfig.QueryParamsMap) == 0 && len(methodConfig.QueryParams) > 0 { + paramsMap, err := parseParamsString2Map(methodConfig.QueryParams) + if err != nil { + logger.Warnf("[Rest Config] Argument Param parse error:%v", err) + } else { + methodConfig.QueryParamsMap = paramsMap + } + } + if len(methodConfig.HeadersMap) == 0 && len(methodConfig.Headers) > 0 { + headersMap, err := parseParamsString2Map(methodConfig.Headers) + if err != nil { + logger.Warnf("[Rest Config] Argument Param parse error:%v", err) + } else { + methodConfig.HeadersMap = headersMap + } + } + return methodConfig +} + +// transform a string to a map +// for example: +// string "0:id,1:name" => map [0:id,1:name] +func parseParamsString2Map(params string) (map[int]string, error) { + m := make(map[int]string, 8) + for _, p := range strings.Split(params, ",") { + pa := strings.Split(p, ":") + key, err := strconv.Atoi(pa[0]) + if err != nil { + return nil, err + } + m[key] = pa[1] + } + return m, nil +} diff --git a/protocol/rest/config/reader/rest_config_reader_test.go b/protocol/rest/config/reader/rest_config_reader_test.go new file mode 100644 index 0000000000..d2dba40b9b --- /dev/null +++ b/protocol/rest/config/reader/rest_config_reader_test.go @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package reader + +import ( + "bytes" + "testing" +) + +import ( + "github.com/stretchr/testify/assert" +) + +import ( + "github.com/apache/dubbo-go/common/yaml" + "github.com/apache/dubbo-go/protocol/rest/config" +) + +func TestRestConfigReader_ReadConsumerConfig(t *testing.T) { + bs, err := yaml.LoadYMLConfig("./testdata/consumer_config.yml") + assert.NoError(t, err) + configReader := NewRestConfigReader() + err = configReader.ReadConsumerConfig(bytes.NewBuffer(bs)) + assert.NoError(t, err) + assert.NotEmpty(t, config.GetRestConsumerServiceConfigMap()) +} + +func TestRestConfigReader_ReadProviderConfig(t *testing.T) { + bs, err := yaml.LoadYMLConfig("./testdata/provider_config.yml") + assert.NoError(t, err) + configReader := NewRestConfigReader() + err = configReader.ReadProviderConfig(bytes.NewBuffer(bs)) + assert.NoError(t, err) + assert.NotEmpty(t, config.GetRestProviderServiceConfigMap()) +} diff --git a/protocol/rest/config/reader/testdata/consumer_config.yml b/protocol/rest/config/reader/testdata/consumer_config.yml new file mode 100644 index 0000000000..27d7fdafef --- /dev/null +++ b/protocol/rest/config/reader/testdata/consumer_config.yml @@ -0,0 +1,74 @@ +# dubbo client yaml configure file + +filter: "" + +config_type: + rest: "rest" + +# client +request_timeout : "100ms" +# connect timeout +connect_timeout : "100ms" +check: true +rest_server: "resty" +rest_produces: "*/*" +rest_consumes: "*/*" + +# application config +application: + organization : "ikurento.com" + name : "BDTService" + module : "dubbogo user-info client" + version : "0.0.1" + owner : "ZX" + environment : "dev" + +registries : + + "hangzhouzk": + protocol: "zookeeper" + timeout : "3s" + address: "127.0.0.1:2181" + username: "" + password: "" + "shanghaizk": + protocol: "zookeeper" + timeout : "3s" + address: "127.0.0.1:2182" + username: "" + password: "" + +references: + "UserProvider": + registry: "hangzhouzk,shanghaizk" + filter: "" + protocol : "rest" + version: "1.0" + group: "as" + interface : "com.ikurento.user.UserProvider" + url: "dubbo://127.0.0.1:20000/UserProvider" + cluster: "failover" + timeout: "3s" + rest_client: "resty1" + rest_produces: "application/xml" + rest_consumes: "application/xml" + methods : + - name: "GetUser" + retries: "3" + timeout: "5s" + rest_query_params: "1:userid,2:username" + rest_headers: "3:age" + rest_path_params: "4:time,2:name" + rest_body: 0 + rest_produces: "application/xml" + rest_consumes: "application/xml" + + params: + "serviceid": + "soa.com.ikurento.user.UserProvider" + "forks": 5 + +shutdown_conf: + timeout: 60s + step_timeout: 10s + diff --git a/protocol/rest/config/reader/testdata/provider_config.yml b/protocol/rest/config/reader/testdata/provider_config.yml new file mode 100644 index 0000000000..71d056e727 --- /dev/null +++ b/protocol/rest/config/reader/testdata/provider_config.yml @@ -0,0 +1,88 @@ +# dubbo server yaml configure file + +filter: "" + +config_type: + rest: "rest" + +# application config +application: + organization : "ikurento.com" + name : "BDTService" + module : "dubbogo user-info server" + version : "0.0.1" + owner : "ZX" + environment : "dev" + +registries : + "hangzhouzk": + protocol: "zookeeper" + timeout : "3s" + address: "127.0.0.1:2181" + username: "" + password: "" + "shanghaizk": + protocol: "zookeeper" + timeout : "3s" + address: "127.0.0.1:2182" + username: "" + password: "" + +rest_server: "go-restful" +rest_produces: "*/*" +rest_consumes: "*/*" + +services: + "UserProvider": + registry: "hangzhouzk,shanghaizk" + filter: "" + # the name of limiter + tps.limiter: "default" + # the time unit of interval is ms + tps.limit.interval: 60000 + tps.limit.rate: 200 + # the name of strategy + tps.limit.strategy: "slidingWindow" + # the name of RejectedExecutionHandler + tps.limit.rejected.handler: "default" + # the concurrent request limitation of this service + # if the value < 0, it will not be limited. + execute.limit: "200" + # the name of RejectedExecutionHandler + execute.limit.rejected.handler: "default" + protocol : "rest" + # equivalent to interface of dubbo.xml + interface : "com.ikurento.user.UserProvider" + loadbalance: "random" + version: "1.0" + group: "as" + warmup: "100" + cluster: "failover" + rest_server: "go-restful1" + rest_produces: "*/*" + rest_consumes: "*/*" + methods: + - name: "GetUser" + retries: 1 + loadbalance: "random" + # the concurrent request limitation of this method + # if the value < 0, it will not be limited. + execute.limit: "200" + # the name of RejectedExecutionHandler + execute.limit.rejected.handler: "default" + rest_query_params: "1:userid,2:username" + rest_headers: "3:age" + rest_path_params: "4:time,2:name" + rest_body: 0 + rest_produces: "application/xml" + rest_consumes: "application/xml" + +protocols: + "rest": + name: "rest" + ip : "127.0.0.1" + port : 20000 + + + + diff --git a/protocol/rest/config/rest_config.go b/protocol/rest/config/rest_config.go new file mode 100644 index 0000000000..168ec8ce52 --- /dev/null +++ b/protocol/rest/config/rest_config.go @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package config + +import ( + "github.com/creasty/defaults" +) + +var ( + restConsumerServiceConfigMap map[string]*RestServiceConfig + restProviderServiceConfigMap map[string]*RestServiceConfig +) + +// RestConsumerConfig ... +type RestConsumerConfig struct { + Client string `default:"resty" yaml:"rest_client" json:"rest_client,omitempty" property:"rest_client"` + Produces string `default:"application/json" yaml:"rest_produces" json:"rest_produces,omitempty" property:"rest_produces"` + Consumes string `default:"application/json" yaml:"rest_consumes" json:"rest_consumes,omitempty" property:"rest_consumes"` + RestServiceConfigsMap map[string]*RestServiceConfig `yaml:"references" json:"references,omitempty" property:"references"` +} + +// UnmarshalYAML ... +func (c *RestConsumerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + if err := defaults.Set(c); err != nil { + return err + } + type plain RestConsumerConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + return nil +} + +// RestProviderConfig ... +type RestProviderConfig struct { + Server string `default:"go-restful" yaml:"rest_server" json:"rest_server,omitempty" property:"rest_server"` + Produces string `default:"*/*" yaml:"rest_produces" json:"rest_produces,omitempty" property:"rest_produces"` + Consumes string `default:"*/*" yaml:"rest_consumes" json:"rest_consumes,omitempty" property:"rest_consumes"` + RestServiceConfigsMap map[string]*RestServiceConfig `yaml:"services" json:"services,omitempty" property:"services"` +} + +// UnmarshalYAML ... +func (c *RestProviderConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + if err := defaults.Set(c); err != nil { + return err + } + type plain RestProviderConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + return nil +} + +// RestServiceConfig ... +type RestServiceConfig struct { + InterfaceName string `required:"true" yaml:"interface" json:"interface,omitempty" property:"interface"` + Url string `yaml:"url" json:"url,omitempty" property:"url"` + Path string `yaml:"rest_path" json:"rest_path,omitempty" property:"rest_path"` + Produces string `yaml:"rest_produces" json:"rest_produces,omitempty" property:"rest_produces"` + Consumes string `yaml:"rest_consumes" json:"rest_consumes,omitempty" property:"rest_consumes"` + MethodType string `yaml:"rest_method" json:"rest_method,omitempty" property:"rest_method"` + Client string `yaml:"rest_client" json:"rest_client,omitempty" property:"rest_client"` + Server string `yaml:"rest_server" json:"rest_server,omitempty" property:"rest_server"` + RestMethodConfigs []*RestMethodConfig `yaml:"methods" json:"methods,omitempty" property:"methods"` + RestMethodConfigsMap map[string]*RestMethodConfig +} + +// UnmarshalYAML ... +func (c *RestServiceConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + if err := defaults.Set(c); err != nil { + return err + } + type plain RestServiceConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + return nil +} + +// RestMethodConfig ... +type RestMethodConfig struct { + InterfaceName string + MethodName string `required:"true" yaml:"name" json:"name,omitempty" property:"name"` + Url string `yaml:"url" json:"url,omitempty" property:"url"` + Path string `yaml:"rest_path" json:"rest_path,omitempty" property:"rest_path"` + Produces string `yaml:"rest_produces" json:"rest_produces,omitempty" property:"rest_produces"` + Consumes string `yaml:"rest_consumes" json:"rest_consumes,omitempty" property:"rest_consumes"` + MethodType string `yaml:"rest_method" json:"rest_method,omitempty" property:"rest_method"` + PathParams string `yaml:"rest_path_params" json:"rest_path_params,omitempty" property:"rest_path_params"` + PathParamsMap map[int]string + QueryParams string `yaml:"rest_query_params" json:"rest_query_params,omitempty" property:"rest_query_params"` + QueryParamsMap map[int]string + Body int `default:"-1" yaml:"rest_body" json:"rest_body,omitempty" property:"rest_body"` + Headers string `yaml:"rest_headers" json:"rest_headers,omitempty" property:"rest_headers"` + HeadersMap map[int]string +} + +// UnmarshalYAML ... +func (c *RestMethodConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + if err := defaults.Set(c); err != nil { + return err + } + type plain RestMethodConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + return nil +} + +// GetRestConsumerServiceConfig ... +func GetRestConsumerServiceConfig(path string) *RestServiceConfig { + return restConsumerServiceConfigMap[path] +} + +// GetRestProviderServiceConfig ... +func GetRestProviderServiceConfig(path string) *RestServiceConfig { + return restProviderServiceConfigMap[path] +} + +// SetRestConsumerServiceConfigMap ... +func SetRestConsumerServiceConfigMap(configMap map[string]*RestServiceConfig) { + restConsumerServiceConfigMap = configMap +} + +// SetRestProviderServiceConfigMap ... +func SetRestProviderServiceConfigMap(configMap map[string]*RestServiceConfig) { + restProviderServiceConfigMap = configMap +} + +// GetRestConsumerServiceConfigMap ... +func GetRestConsumerServiceConfigMap() map[string]*RestServiceConfig { + return restConsumerServiceConfigMap +} + +// GetRestProviderServiceConfigMap ... +func GetRestProviderServiceConfigMap() map[string]*RestServiceConfig { + return restProviderServiceConfigMap +} diff --git a/protocol/rest/rest_exporter.go b/protocol/rest/rest_exporter.go new file mode 100644 index 0000000000..470d525ad8 --- /dev/null +++ b/protocol/rest/rest_exporter.go @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rest + +import ( + "sync" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/common/constant" + "github.com/apache/dubbo-go/common/logger" + "github.com/apache/dubbo-go/protocol" +) + +type RestExporter struct { + protocol.BaseExporter +} + +func NewRestExporter(key string, invoker protocol.Invoker, exporterMap *sync.Map) *RestExporter { + return &RestExporter{ + BaseExporter: *protocol.NewBaseExporter(key, invoker, exporterMap), + } +} + +func (re *RestExporter) Unexport() { + serviceId := re.GetInvoker().GetUrl().GetParam(constant.BEAN_NAME_KEY, "") + re.BaseExporter.Unexport() + err := common.ServiceMap.UnRegister(REST, serviceId) + if err != nil { + logger.Errorf("[RestExporter.Unexport] error: %v", err) + } + return +} diff --git a/protocol/rest/rest_invoker.go b/protocol/rest/rest_invoker.go new file mode 100644 index 0000000000..0c82035ac5 --- /dev/null +++ b/protocol/rest/rest_invoker.go @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rest + +import ( + "context" + "fmt" +) + +import ( + perrors "github.com/pkg/errors" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/protocol" + invocation_impl "github.com/apache/dubbo-go/protocol/invocation" + "github.com/apache/dubbo-go/protocol/rest/client" + "github.com/apache/dubbo-go/protocol/rest/config" +) + +type RestInvoker struct { + protocol.BaseInvoker + client client.RestClient + restMethodConfigMap map[string]*config.RestMethodConfig +} + +func NewRestInvoker(url common.URL, client *client.RestClient, restMethodConfig map[string]*config.RestMethodConfig) *RestInvoker { + return &RestInvoker{ + BaseInvoker: *protocol.NewBaseInvoker(url), + client: *client, + restMethodConfigMap: restMethodConfig, + } +} + +func (ri *RestInvoker) Invoke(ctx context.Context, invocation protocol.Invocation) protocol.Result { + inv := invocation.(*invocation_impl.RPCInvocation) + methodConfig := ri.restMethodConfigMap[inv.MethodName()] + var ( + result protocol.RPCResult + body interface{} + pathParams map[string]string + queryParams map[string]string + headers map[string]string + err error + ) + if methodConfig == nil { + result.Err = perrors.Errorf("[RestInvoker] Rest methodConfig:%s is nil", inv.MethodName()) + return &result + } + if pathParams, err = restStringMapTransform(methodConfig.PathParamsMap, inv.Arguments()); err != nil { + result.Err = err + return &result + } + if queryParams, err = restStringMapTransform(methodConfig.QueryParamsMap, inv.Arguments()); err != nil { + result.Err = err + return &result + } + if headers, err = restStringMapTransform(methodConfig.HeadersMap, inv.Arguments()); err != nil { + result.Err = err + return &result + } + if len(inv.Arguments()) > methodConfig.Body && methodConfig.Body >= 0 { + body = inv.Arguments()[methodConfig.Body] + } + + req := &client.RestRequest{ + Location: ri.GetUrl().Location, + Produces: methodConfig.Produces, + Consumes: methodConfig.Consumes, + Method: methodConfig.MethodType, + Path: methodConfig.Path, + PathParams: pathParams, + QueryParams: queryParams, + Body: body, + Headers: headers, + } + result.Err = ri.client.Do(req, inv.Reply()) + if result.Err == nil { + result.Rest = inv.Reply() + } + return &result +} + +func restStringMapTransform(paramsMap map[int]string, args []interface{}) (map[string]string, error) { + resMap := make(map[string]string, len(paramsMap)) + for k, v := range paramsMap { + if k >= len(args) || k < 0 { + return nil, perrors.Errorf("[Rest Invoke] Index %v is out of bundle", k) + } + resMap[v] = fmt.Sprint(args[k]) + } + return resMap, nil +} diff --git a/protocol/rest/rest_invoker_test.go b/protocol/rest/rest_invoker_test.go new file mode 100644 index 0000000000..42a4fbd358 --- /dev/null +++ b/protocol/rest/rest_invoker_test.go @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rest + +import ( + "context" + "testing" + "time" +) + +import ( + "github.com/stretchr/testify/assert" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/common/extension" + "github.com/apache/dubbo-go/config" + "github.com/apache/dubbo-go/protocol/invocation" + "github.com/apache/dubbo-go/protocol/rest/client" + "github.com/apache/dubbo-go/protocol/rest/client/client_impl" + rest_config "github.com/apache/dubbo-go/protocol/rest/config" +) + +func TestRestInvoker_Invoke(t *testing.T) { + // Refer + proto := GetRestProtocol() + defer proto.Destroy() + url, err := common.NewURL("rest://127.0.0.1:8877/com.ikurento.user.UserProvider?anyhost=true&" + + "application=BDTService&category=providers&default.timeout=10000&dubbo=dubbo-provider-golang-1.0.0&" + + "environment=dev&interface=com.ikurento.user.UserProvider&ip=192.168.56.1&methods=GetUser%2C&" + + "module=dubbogo+user-info+server&org=ikurento.com&owner=ZX&pid=1447&revision=0.0.1&" + + "side=provider&timeout=3000×tamp=1556509797245") + assert.NoError(t, err) + _, err = common.ServiceMap.Register(url.Protocol, &UserProvider{}) + assert.NoError(t, err) + con := config.ProviderConfig{} + config.SetProviderConfig(con) + configMap := make(map[string]*rest_config.RestServiceConfig) + methodConfigMap := make(map[string]*rest_config.RestMethodConfig) + queryParamsMap := make(map[int]string) + queryParamsMap[1] = "age" + queryParamsMap[2] = "name" + pathParamsMap := make(map[int]string) + pathParamsMap[0] = "userid" + headersMap := make(map[int]string) + headersMap[3] = "Content-Type" + methodConfigMap["GetUserOne"] = &rest_config.RestMethodConfig{ + InterfaceName: "", + MethodName: "GetUserOne", + Path: "/GetUserOne", + Produces: "application/json", + Consumes: "application/json", + MethodType: "POST", + PathParams: "", + PathParamsMap: nil, + QueryParams: "", + QueryParamsMap: nil, + Body: 0, + } + methodConfigMap["GetUserTwo"] = &rest_config.RestMethodConfig{ + InterfaceName: "", + MethodName: "GetUserTwo", + Path: "/GetUserTwo", + Produces: "application/json", + Consumes: "application/json", + MethodType: "POST", + PathParams: "", + PathParamsMap: nil, + QueryParams: "", + QueryParamsMap: nil, + Body: 0, + } + methodConfigMap["GetUserThree"] = &rest_config.RestMethodConfig{ + InterfaceName: "", + MethodName: "GetUserThree", + Path: "/GetUserThree", + Produces: "application/json", + Consumes: "application/json", + MethodType: "POST", + PathParams: "", + PathParamsMap: nil, + QueryParams: "", + QueryParamsMap: nil, + Body: 0, + } + methodConfigMap["GetUserFour"] = &rest_config.RestMethodConfig{ + InterfaceName: "", + MethodName: "GetUserFour", + Path: "/GetUserFour", + Produces: "application/json", + Consumes: "application/json", + MethodType: "POST", + PathParams: "", + PathParamsMap: nil, + QueryParams: "", + QueryParamsMap: nil, + Body: 0, + } + methodConfigMap["GetUserFive"] = &rest_config.RestMethodConfig{ + InterfaceName: "", + MethodName: "GetUserFive", + Path: "/GetUserFive", + Produces: "*/*", + Consumes: "*/*", + MethodType: "GET", + } + methodConfigMap["GetUser"] = &rest_config.RestMethodConfig{ + InterfaceName: "", + MethodName: "GetUser", + Path: "/GetUser/{userid}", + Produces: "application/json", + Consumes: "application/json", + MethodType: "GET", + PathParams: "", + PathParamsMap: pathParamsMap, + QueryParams: "", + QueryParamsMap: queryParamsMap, + Body: -1, + HeadersMap: headersMap, + } + + configMap["com.ikurento.user.UserProvider"] = &rest_config.RestServiceConfig{ + Server: "go-restful", + RestMethodConfigsMap: methodConfigMap, + } + rest_config.SetRestProviderServiceConfigMap(configMap) + proxyFactory := extension.GetProxyFactory("default") + proto.Export(proxyFactory.GetInvoker(url)) + time.Sleep(5 * time.Second) + configMap = make(map[string]*rest_config.RestServiceConfig) + configMap["com.ikurento.user.UserProvider"] = &rest_config.RestServiceConfig{ + RestMethodConfigsMap: methodConfigMap, + } + restClient := client_impl.NewRestyClient(&client.RestOptions{ConnectTimeout: 3 * time.Second, RequestTimeout: 3 * time.Second}) + invoker := NewRestInvoker(url, &restClient, methodConfigMap) + user := &User{} + inv := invocation.NewRPCInvocationWithOptions(invocation.WithMethodName("GetUser"), + invocation.WithArguments([]interface{}{1, int32(23), "username", "application/json"}), invocation.WithReply(user)) + res := invoker.Invoke(context.Background(), inv) + assert.NoError(t, res.Error()) + assert.Equal(t, User{Id: 1, Age: int32(23), Name: "username"}, *res.Result().(*User)) + now := time.Now() + inv = invocation.NewRPCInvocationWithOptions(invocation.WithMethodName("GetUserOne"), + invocation.WithArguments([]interface{}{&User{1, &now, int32(23), "username"}}), invocation.WithReply(user)) + res = invoker.Invoke(context.Background(), inv) + assert.NoError(t, res.Error()) + assert.NotNil(t, res.Result()) + assert.Equal(t, 1, res.Result().(*User).Id) + assert.Equal(t, now.Unix(), res.Result().(*User).Time.Unix()) + assert.Equal(t, int32(23), res.Result().(*User).Age) + assert.Equal(t, "username", res.Result().(*User).Name) + // test 1 + inv = invocation.NewRPCInvocationWithOptions(invocation.WithMethodName("GetUserTwo"), + invocation.WithArguments([]interface{}{&User{1, &now, int32(23), "username"}}), invocation.WithReply(user)) + res = invoker.Invoke(context.Background(), inv) + assert.NoError(t, res.Error()) + assert.NotNil(t, res.Result()) + assert.Equal(t, "username", res.Result().(*User).Name) + // test 2 + inv = invocation.NewRPCInvocationWithOptions(invocation.WithMethodName("GetUserThree"), + invocation.WithArguments([]interface{}{&User{1, &now, int32(23), "username"}}), invocation.WithReply(user)) + res = invoker.Invoke(context.Background(), inv) + assert.NoError(t, res.Error()) + assert.NotNil(t, res.Result()) + assert.Equal(t, "username", res.Result().(*User).Name) + // test 3 + inv = invocation.NewRPCInvocationWithOptions(invocation.WithMethodName("GetUserFour"), + invocation.WithArguments([]interface{}{[]User{User{1, nil, int32(23), "username"}}}), invocation.WithReply(user)) + res = invoker.Invoke(context.Background(), inv) + assert.NoError(t, res.Error()) + assert.NotNil(t, res.Result()) + assert.Equal(t, "username", res.Result().(*User).Name) + // test 4 + inv = invocation.NewRPCInvocationWithOptions(invocation.WithMethodName("GetUserFive"), invocation.WithReply(user)) + res = invoker.Invoke(context.Background(), inv) + assert.Error(t, res.Error(), "test error") + + err = common.ServiceMap.UnRegister(url.Protocol, "com.ikurento.user.UserProvider") + assert.NoError(t, err) +} diff --git a/protocol/rest/rest_protocol.go b/protocol/rest/rest_protocol.go new file mode 100644 index 0000000000..47ecb6093b --- /dev/null +++ b/protocol/rest/rest_protocol.go @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rest + +import ( + "strings" + "sync" + "time" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/common/constant" + "github.com/apache/dubbo-go/common/extension" + "github.com/apache/dubbo-go/common/logger" + "github.com/apache/dubbo-go/config" + "github.com/apache/dubbo-go/protocol" + "github.com/apache/dubbo-go/protocol/rest/client" + _ "github.com/apache/dubbo-go/protocol/rest/client/client_impl" + rest_config "github.com/apache/dubbo-go/protocol/rest/config" + _ "github.com/apache/dubbo-go/protocol/rest/config/reader" + "github.com/apache/dubbo-go/protocol/rest/server" + _ "github.com/apache/dubbo-go/protocol/rest/server/server_impl" +) + +var ( + restProtocol *RestProtocol +) + +const REST = "rest" + +func init() { + extension.SetProtocol(REST, GetRestProtocol) +} + +type RestProtocol struct { + protocol.BaseProtocol + serverLock sync.Mutex + serverMap map[string]server.RestServer + clientLock sync.Mutex + clientMap map[client.RestOptions]client.RestClient +} + +func NewRestProtocol() *RestProtocol { + return &RestProtocol{ + BaseProtocol: protocol.NewBaseProtocol(), + serverMap: make(map[string]server.RestServer, 8), + clientMap: make(map[client.RestOptions]client.RestClient, 8), + } +} + +func (rp *RestProtocol) Export(invoker protocol.Invoker) protocol.Exporter { + url := invoker.GetUrl() + serviceKey := url.ServiceKey() + exporter := NewRestExporter(serviceKey, invoker, rp.ExporterMap()) + restServiceConfig := rest_config.GetRestProviderServiceConfig(strings.TrimPrefix(url.Path, "/")) + if restServiceConfig == nil { + logger.Errorf("%s service doesn't has provider config", url.Path) + return nil + } + rp.SetExporterMap(serviceKey, exporter) + restServer := rp.getServer(url, restServiceConfig.Server) + restServer.Deploy(invoker, restServiceConfig.RestMethodConfigsMap) + return exporter +} + +func (rp *RestProtocol) Refer(url common.URL) protocol.Invoker { + // create rest_invoker + var requestTimeout = config.GetConsumerConfig().RequestTimeout + requestTimeoutStr := url.GetParam(constant.TIMEOUT_KEY, config.GetConsumerConfig().Request_Timeout) + connectTimeout := config.GetConsumerConfig().ConnectTimeout + if t, err := time.ParseDuration(requestTimeoutStr); err == nil { + requestTimeout = t + } + restServiceConfig := rest_config.GetRestConsumerServiceConfig(strings.TrimPrefix(url.Path, "/")) + if restServiceConfig == nil { + logger.Errorf("%s service doesn't has consumer config", url.Path) + return nil + } + restOptions := client.RestOptions{RequestTimeout: requestTimeout, ConnectTimeout: connectTimeout} + restClient := rp.getClient(restOptions, restServiceConfig.Client) + invoker := NewRestInvoker(url, &restClient, restServiceConfig.RestMethodConfigsMap) + rp.SetInvokers(invoker) + return invoker +} + +func (rp *RestProtocol) getServer(url common.URL, serverType string) server.RestServer { + restServer, ok := rp.serverMap[url.Location] + if ok { + return restServer + } + _, ok = rp.ExporterMap().Load(url.ServiceKey()) + if !ok { + panic("[RestProtocol]" + url.ServiceKey() + "is not existing") + } + rp.serverLock.Lock() + defer rp.serverLock.Unlock() + restServer, ok = rp.serverMap[url.Location] + if ok { + return restServer + } + restServer = extension.GetNewRestServer(serverType) + restServer.Start(url) + rp.serverMap[url.Location] = restServer + return restServer +} + +func (rp *RestProtocol) getClient(restOptions client.RestOptions, clientType string) client.RestClient { + restClient, ok := rp.clientMap[restOptions] + if ok { + return restClient + } + rp.clientLock.Lock() + defer rp.clientLock.Unlock() + restClient, ok = rp.clientMap[restOptions] + if ok { + return restClient + } + restClient = extension.GetNewRestClient(clientType, &restOptions) + rp.clientMap[restOptions] = restClient + return restClient +} + +func (rp *RestProtocol) Destroy() { + // destroy rest_server + rp.BaseProtocol.Destroy() + for key, server := range rp.serverMap { + server.Destroy() + delete(rp.serverMap, key) + } + for key := range rp.clientMap { + delete(rp.clientMap, key) + } +} + +func GetRestProtocol() protocol.Protocol { + if restProtocol == nil { + restProtocol = NewRestProtocol() + } + return restProtocol +} diff --git a/protocol/rest/rest_protocol_test.go b/protocol/rest/rest_protocol_test.go new file mode 100644 index 0000000000..8af73a1839 --- /dev/null +++ b/protocol/rest/rest_protocol_test.go @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rest + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" +) + +import ( + "github.com/stretchr/testify/assert" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/common/extension" + _ "github.com/apache/dubbo-go/common/proxy/proxy_factory" + "github.com/apache/dubbo-go/config" + rest_config "github.com/apache/dubbo-go/protocol/rest/config" +) + +func TestRestProtocol_Refer(t *testing.T) { + // Refer + proto := GetRestProtocol() + url, err := common.NewURL("rest://127.0.0.1:20000/com.ikurento.user.UserProvider?anyhost=true&" + + "application=BDTService&category=providers&default.timeout=10000&dubbo=dubbo-provider-golang-1.0.0&" + + "environment=dev&interface=com.ikurento.user.UserProvider&ip=192.168.56.1&methods=GetUser%2C&" + + "module=dubbogo+user-info+server&org=ikurento.com&owner=ZX&pid=1447&revision=0.0.1&" + + "side=provider&timeout=3000×tamp=1556509797245") + assert.NoError(t, err) + con := config.ConsumerConfig{ + ConnectTimeout: 5 * time.Second, + RequestTimeout: 5 * time.Second, + } + config.SetConsumerConfig(con) + configMap := make(map[string]*rest_config.RestServiceConfig) + configMap["com.ikurento.user.UserProvider"] = &rest_config.RestServiceConfig{ + Client: "resty", + } + rest_config.SetRestConsumerServiceConfigMap(configMap) + invoker := proto.Refer(url) + + // make sure url + eq := invoker.GetUrl().URLEqual(url) + assert.True(t, eq) + + // make sure invokers after 'Destroy' + invokersLen := len(proto.(*RestProtocol).Invokers()) + assert.Equal(t, 1, invokersLen) + proto.Destroy() + invokersLen = len(proto.(*RestProtocol).Invokers()) + assert.Equal(t, 0, invokersLen) +} + +func TestRestProtocol_Export(t *testing.T) { + // Export + proto := GetRestProtocol() + url, err := common.NewURL("rest://127.0.0.1:8888/com.ikurento.user.UserProvider?anyhost=true&" + + "application=BDTService&category=providers&default.timeout=10000&dubbo=dubbo-provider-golang-1.0.0&" + + "environment=dev&interface=com.ikurento.user.UserProvider&ip=192.168.56.1&methods=GetUser%2C&" + + "module=dubbogo+user-info+server&org=ikurento.com&owner=ZX&pid=1447&revision=0.0.1&" + + "side=provider&timeout=3000×tamp=1556509797245") + assert.NoError(t, err) + _, err = common.ServiceMap.Register(url.Protocol, &UserProvider{}) + assert.NoError(t, err) + con := config.ProviderConfig{} + config.SetProviderConfig(con) + configMap := make(map[string]*rest_config.RestServiceConfig) + methodConfigMap := make(map[string]*rest_config.RestMethodConfig) + queryParamsMap := make(map[int]string) + queryParamsMap[1] = "age" + queryParamsMap[2] = "name" + pathParamsMap := make(map[int]string) + pathParamsMap[0] = "userid" + methodConfigMap["GetUser"] = &rest_config.RestMethodConfig{ + InterfaceName: "", + MethodName: "GetUser", + Path: "/GetUser/{userid}", + Produces: "application/json", + Consumes: "application/json", + MethodType: "GET", + PathParams: "", + PathParamsMap: pathParamsMap, + QueryParams: "", + QueryParamsMap: queryParamsMap, + Body: -1, + } + configMap["com.ikurento.user.UserProvider"] = &rest_config.RestServiceConfig{ + Server: "go-restful", + RestMethodConfigsMap: methodConfigMap, + } + rest_config.SetRestProviderServiceConfigMap(configMap) + proxyFactory := extension.GetProxyFactory("default") + exporter := proto.Export(proxyFactory.GetInvoker(url)) + // make sure url + eq := exporter.GetInvoker().GetUrl().URLEqual(url) + assert.True(t, eq) + // make sure exporterMap after 'Unexport' + fmt.Println(url.Path) + _, ok := proto.(*RestProtocol).ExporterMap().Load(strings.TrimPrefix(url.Path, "/")) + assert.True(t, ok) + exporter.Unexport() + _, ok = proto.(*RestProtocol).ExporterMap().Load(strings.TrimPrefix(url.Path, "/")) + assert.False(t, ok) + + // make sure serverMap after 'Destroy' + _, ok = proto.(*RestProtocol).serverMap[url.Location] + assert.True(t, ok) + proto.Destroy() + _, ok = proto.(*RestProtocol).serverMap[url.Location] + assert.False(t, ok) + err = common.ServiceMap.UnRegister(url.Protocol, "com.ikurento.user.UserProvider") + assert.NoError(t, err) +} + +type UserProvider struct { +} + +func (p *UserProvider) Reference() string { + return "com.ikurento.user.UserProvider" +} + +func (p *UserProvider) GetUser(ctx context.Context, id int, age int32, name string, contentType string) (*User, error) { + return &User{ + Id: id, + Time: nil, + Age: age, + Name: name, + }, nil +} + +func (p *UserProvider) GetUserOne(ctx context.Context, user *User) (*User, error) { + return user, nil +} + +func (p *UserProvider) GetUserTwo(ctx context.Context, req []interface{}, rsp *User) error { + m := req[0].(map[string]interface{}) + rsp.Name = m["Name"].(string) + return nil +} + +func (p *UserProvider) GetUserThree(ctx context.Context, user interface{}) (*User, error) { + m := user.(map[string]interface{}) + + u := &User{} + u.Name = m["Name"].(string) + return u, nil +} + +func (p *UserProvider) GetUserFour(ctx context.Context, user []interface{}, id string) (*User, error) { + m := user[0].(map[string]interface{}) + + u := &User{} + u.Name = m["Name"].(string) + return u, nil +} + +func (p *UserProvider) GetUserFive(ctx context.Context, user []interface{}) (*User, error) { + return nil, errors.New("test error") +} + +type User struct { + Id int + Time *time.Time + Age int32 + Name string +} diff --git a/protocol/rest/server/rest_server.go b/protocol/rest/server/rest_server.go new file mode 100644 index 0000000000..c10c98a7b6 --- /dev/null +++ b/protocol/rest/server/rest_server.go @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package server + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/protocol" + "github.com/apache/dubbo-go/protocol/rest/config" +) + +type RestServer interface { + Start(url common.URL) + Deploy(invoker protocol.Invoker, restMethodConfig map[string]*config.RestMethodConfig) + UnDeploy(restMethodConfig map[string]*config.RestMethodConfig) + Destroy() +} diff --git a/protocol/rest/server/server_impl/go_restful_server.go b/protocol/rest/server/server_impl/go_restful_server.go new file mode 100644 index 0000000000..3ea25531d6 --- /dev/null +++ b/protocol/rest/server/server_impl/go_restful_server.go @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package server_impl + +import ( + "context" + "fmt" + "net" + "net/http" + "reflect" + "strconv" + "strings" + "time" +) + +import ( + "github.com/emicklei/go-restful/v3" + perrors "github.com/pkg/errors" +) + +import ( + "github.com/apache/dubbo-go/common" + "github.com/apache/dubbo-go/common/constant" + "github.com/apache/dubbo-go/common/extension" + "github.com/apache/dubbo-go/common/logger" + "github.com/apache/dubbo-go/protocol" + "github.com/apache/dubbo-go/protocol/invocation" + "github.com/apache/dubbo-go/protocol/rest/config" + "github.com/apache/dubbo-go/protocol/rest/server" +) + +func init() { + extension.SetRestServer(constant.DEFAULT_REST_SERVER, GetNewGoRestfulServer) +} + +type GoRestfulServer struct { + srv *http.Server + container *restful.Container +} + +func NewGoRestfulServer() *GoRestfulServer { + return &GoRestfulServer{} +} + +func (grs *GoRestfulServer) Start(url common.URL) { + grs.container = restful.NewContainer() + grs.srv = &http.Server{ + Handler: grs.container, + } + ln, err := net.Listen("tcp", url.Location) + if err != nil { + panic(perrors.New(fmt.Sprintf("Restful Server start error:%v", err))) + } + + go func() { + err := grs.srv.Serve(ln) + if err != nil && err != http.ErrServerClosed { + logger.Errorf("[Go Restful] http.server.Serve(addr{%s}) = err{%+v}", url.Location, err) + } + }() +} + +func (grs *GoRestfulServer) Deploy(invoker protocol.Invoker, restMethodConfig map[string]*config.RestMethodConfig) { + svc := common.ServiceMap.GetService(invoker.GetUrl().Protocol, strings.TrimPrefix(invoker.GetUrl().Path, "/")) + for methodName, config := range restMethodConfig { + // get method + method := svc.Method()[methodName] + argsTypes := method.ArgsType() + replyType := method.ReplyType() + ws := new(restful.WebService) + ws.Path(config.Path). + Produces(strings.Split(config.Produces, ",")...). + Consumes(strings.Split(config.Consumes, ",")...). + Route(ws.Method(config.MethodType).To(getFunc(methodName, invoker, argsTypes, replyType, config))) + grs.container.Add(ws) + } + +} + +func getFunc(methodName string, invoker protocol.Invoker, argsTypes []reflect.Type, + replyType reflect.Type, config *config.RestMethodConfig) func(req *restful.Request, resp *restful.Response) { + return func(req *restful.Request, resp *restful.Response) { + var ( + err error + args []interface{} + ) + if (len(argsTypes) == 1 || len(argsTypes) == 2 && replyType == nil) && + argsTypes[0].String() == "[]interface {}" { + args = getArgsInterfaceFromRequest(req, config) + } else { + args = getArgsFromRequest(req, argsTypes, config) + } + result := invoker.Invoke(context.Background(), invocation.NewRPCInvocation(methodName, args, make(map[string]string))) + if result.Error() != nil { + err = resp.WriteError(http.StatusInternalServerError, result.Error()) + if err != nil { + logger.Errorf("[Go Restful] WriteError error:%v", err) + } + return + } + err = resp.WriteEntity(result.Result()) + if err != nil { + logger.Error("[Go Restful] WriteEntity error:%v", err) + } + } +} +func (grs *GoRestfulServer) UnDeploy(restMethodConfig map[string]*config.RestMethodConfig) { + for _, config := range restMethodConfig { + ws := new(restful.WebService) + ws.Path(config.Path) + err := grs.container.Remove(ws) + if err != nil { + logger.Warnf("[Go restful] Remove web service error:%v", err) + } + } +} + +func (grs *GoRestfulServer) Destroy() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := grs.srv.Shutdown(ctx); err != nil { + logger.Errorf("[Go Restful] Server Shutdown:", err) + } + logger.Infof("[Go Restful] Server exiting") +} + +func getArgsInterfaceFromRequest(req *restful.Request, config *config.RestMethodConfig) []interface{} { + argsMap := make(map[int]interface{}, 8) + maxKey := 0 + for k, v := range config.PathParamsMap { + if maxKey < k { + maxKey = k + } + argsMap[k] = req.PathParameter(v) + } + for k, v := range config.QueryParamsMap { + if maxKey < k { + maxKey = k + } + params := req.QueryParameters(v) + if len(params) == 1 { + argsMap[k] = params[0] + } else { + argsMap[k] = params + } + } + for k, v := range config.HeadersMap { + if maxKey < k { + maxKey = k + } + argsMap[k] = req.HeaderParameter(v) + } + if config.Body >= 0 { + if maxKey < config.Body { + maxKey = config.Body + } + m := make(map[string]interface{}) + // TODO read as a slice + if err := req.ReadEntity(&m); err != nil { + logger.Warnf("[Go restful] Read body entity as map[string]interface{} error:%v", perrors.WithStack(err)) + } else { + argsMap[config.Body] = m + } + } + args := make([]interface{}, maxKey+1) + for k, v := range argsMap { + if k >= 0 { + args[k] = v + } + } + return args +} + +func getArgsFromRequest(req *restful.Request, argsTypes []reflect.Type, config *config.RestMethodConfig) []interface{} { + argsLength := len(argsTypes) + args := make([]interface{}, argsLength) + for i, t := range argsTypes { + args[i] = reflect.Zero(t).Interface() + } + var ( + err error + param interface{} + i64 int64 + ) + for k, v := range config.PathParamsMap { + if k < 0 || k >= argsLength { + logger.Errorf("[Go restful] Path param parse error, the args:%v doesn't exist", k) + continue + } + t := argsTypes[k] + kind := t.Kind() + if kind == reflect.Ptr { + t = t.Elem() + } + if kind == reflect.Int { + param, err = strconv.Atoi(req.PathParameter(v)) + } else if kind == reflect.Int32 { + i64, err = strconv.ParseInt(req.PathParameter(v), 10, 32) + if err == nil { + param = int32(i64) + } + } else if kind == reflect.Int64 { + param, err = strconv.ParseInt(req.PathParameter(v), 10, 64) + } else if kind == reflect.String { + param = req.PathParameter(v) + } else { + logger.Warnf("[Go restful] Path param parse error, the args:%v of type isn't int or string", k) + continue + } + if err != nil { + logger.Errorf("[Go restful] Path param parse error, error is %v", err) + continue + } + args[k] = param + } + for k, v := range config.QueryParamsMap { + if k < 0 || k >= argsLength { + logger.Errorf("[Go restful] Query param parse error, the args:%v doesn't exist", k) + continue + } + t := argsTypes[k] + kind := t.Kind() + if kind == reflect.Ptr { + t = t.Elem() + } + if kind == reflect.Slice { + param = req.QueryParameters(v) + } else if kind == reflect.String { + param = req.QueryParameter(v) + } else if kind == reflect.Int { + param, err = strconv.Atoi(req.QueryParameter(v)) + } else if kind == reflect.Int32 { + i64, err = strconv.ParseInt(req.QueryParameter(v), 10, 32) + if err == nil { + param = int32(i64) + } + } else if kind == reflect.Int64 { + param, err = strconv.ParseInt(req.QueryParameter(v), 10, 64) + } else { + logger.Errorf("[Go restful] Query param parse error, the args:%v of type isn't int or string or slice", k) + continue + } + if err != nil { + logger.Errorf("[Go restful] Query param parse error, error is %v", err) + continue + } + args[k] = param + } + + if config.Body >= 0 && config.Body < len(argsTypes) { + t := argsTypes[config.Body] + kind := t.Kind() + if kind == reflect.Ptr { + t = t.Elem() + } + var ni interface{} + if t.String() == "[]interface {}" { + ni = make([]map[string]interface{}, 0) + } else if t.String() == "interface {}" { + ni = make(map[string]interface{}) + } else { + n := reflect.New(t) + if n.CanInterface() { + ni = n.Interface() + } + } + if err := req.ReadEntity(&ni); err != nil { + logger.Errorf("[Go restful] Read body entity error:%v", err) + } else { + args[config.Body] = ni + } + } + + for k, v := range config.HeadersMap { + param := req.HeaderParameter(v) + if k < 0 || k >= argsLength { + logger.Errorf("[Go restful] Header param parse error, the args:%v doesn't exist", k) + continue + } + t := argsTypes[k] + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() == reflect.String { + args[k] = param + } else { + logger.Errorf("[Go restful] Header param parse error, the args:%v of type isn't string", k) + } + } + + return args +} + +func GetNewGoRestfulServer() server.RestServer { + return NewGoRestfulServer() +} diff --git a/registry/base_registry.go b/registry/base_registry.go index 5b9aef8292..3b64e93e2f 100644 --- a/registry/base_registry.go +++ b/registry/base_registry.go @@ -69,11 +69,20 @@ func init() { */ type FacadeBasedRegistry interface { Registry + + // CreatePath create the path in the registry CreatePath(string) error + // DoRegister actually do the register job DoRegister(string, string) error + // DoSubscribe actually subscribe the URL DoSubscribe(conf *common.URL) (Listener, error) + // CloseAndNilClient close the client and then reset the client in registry to nil + // you should notice that this method will be invoked inside a lock. + // So you should implement this method as light weighted as you can. CloseAndNilClient() + // CloseListener close listeners CloseListener() + // InitListeners init listeners InitListeners() } @@ -153,7 +162,7 @@ func (r *BaseRegistry) service(c common.URL) string { func (r *BaseRegistry) RestartCallBack() bool { // copy r.services - services := []common.URL{} + services := make([]common.URL, 0, len(r.services)) for _, confIf := range r.services { services = append(services, confIf) } @@ -227,9 +236,11 @@ func (r *BaseRegistry) providerRegistry(c common.URL, params url.Values) (string return "", "", perrors.Errorf("conf{Path:%s, Methods:%s}", c.Path, c.Methods) } dubboPath = fmt.Sprintf("/dubbo/%s/%s", r.service(c), common.DubboNodes[common.PROVIDER]) - r.cltLock.Lock() - err = r.facadeBasedRegistry.CreatePath(dubboPath) - r.cltLock.Unlock() + func() { + r.cltLock.Lock() + defer r.cltLock.Unlock() + err = r.facadeBasedRegistry.CreatePath(dubboPath) + }() if err != nil { logger.Errorf("facadeBasedRegistry.CreatePath(path{%s}) = error{%#v}", dubboPath, perrors.WithStack(err)) return "", "", perrors.WithMessagef(err, "facadeBasedRegistry.CreatePath(path:%s)", dubboPath) @@ -251,10 +262,11 @@ func (r *BaseRegistry) providerRegistry(c common.URL, params url.Values) (string logger.Debugf("provider url params:%#v", params) var host string if c.Ip == "" { - host = localIP + ":" + c.Port + host = localIP } else { - host = c.Ip + ":" + c.Port + host = c.Ip } + host += ":" + c.Port rawURL = fmt.Sprintf("%s://%s%s?%s", c.Protocol, host, c.Path, params.Encode()) // Print your own registration service providers. @@ -271,17 +283,25 @@ func (r *BaseRegistry) consumerRegistry(c common.URL, params url.Values) (string err error ) dubboPath = fmt.Sprintf("/dubbo/%s/%s", r.service(c), common.DubboNodes[common.CONSUMER]) - r.cltLock.Lock() - err = r.facadeBasedRegistry.CreatePath(dubboPath) - r.cltLock.Unlock() + + func() { + r.cltLock.Lock() + defer r.cltLock.Unlock() + err = r.facadeBasedRegistry.CreatePath(dubboPath) + + }() if err != nil { logger.Errorf("facadeBasedRegistry.CreatePath(path{%s}) = error{%v}", dubboPath, perrors.WithStack(err)) return "", "", perrors.WithStack(err) } dubboPath = fmt.Sprintf("/dubbo/%s/%s", r.service(c), common.DubboNodes[common.PROVIDER]) - r.cltLock.Lock() - err = r.facadeBasedRegistry.CreatePath(dubboPath) - r.cltLock.Unlock() + + func() { + r.cltLock.Lock() + defer r.cltLock.Unlock() + err = r.facadeBasedRegistry.CreatePath(dubboPath) + }() + if err != nil { logger.Errorf("facadeBasedRegistry.CreatePath(path{%s}) = error{%v}", dubboPath, perrors.WithStack(err)) return "", "", perrors.WithStack(err) @@ -345,9 +365,9 @@ func (r *BaseRegistry) Subscribe(url *common.URL, notifyListener NotifyListener) // closeRegisters close and remove registry client and reset services map func (r *BaseRegistry) closeRegisters() { + logger.Infof("begin to close provider client") r.cltLock.Lock() defer r.cltLock.Unlock() - logger.Infof("begin to close provider client") // Close and remove(set to nil) the registry client r.facadeBasedRegistry.CloseAndNilClient() // reset the services map From 46f5c9a3e69bd0d462ba954322adaab4093faed8 Mon Sep 17 00:00:00 2001 From: scott Date: Mon, 16 Mar 2020 00:19:01 +0800 Subject: [PATCH 47/61] Fix CHANGE.md --- CHANGE.md | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/CHANGE.md b/CHANGE.md index 8b16870ef8..831a50ce7e 100644 --- a/CHANGE.md +++ b/CHANGE.md @@ -100,24 +100,6 @@ - [Fix bug in reference_config](https://github.com/apache/dubbo-go/pull/157) - [Fix high memory bug in zookeeper listener](https://github.com/apache/dubbo-go/pull/168) -## 1.2.0 - -### New Features - -- Add etcdv3 registry support -- Add nacos registry support -- Add fail fast cluster support -- Add available cluster support -- Add broadcast cluster support -- Add forking cluster support -- Add service token authorization support -- Add accessLog filter support -- Add tps limit support -- Add execute limit support -- Move callService to invoker & support attachments -- Move example in dubbo-go project away -- Support dynamic config center which compatible with dubbo 2.6.x & 2.7.x and commit the zookeeper impl - ### Enhancement - Split gettyRPCClient.close and gettyRPCClientPool.remove in protocol/dubbo/pool.go From d7a37bad934c670b196f44390ef27bfc199f3db7 Mon Sep 17 00:00:00 2001 From: scott Date: Mon, 16 Mar 2020 00:24:02 +0800 Subject: [PATCH 48/61] Fix CHANGE.md --- CHANGE.md | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/CHANGE.md b/CHANGE.md index 831a50ce7e..ad8bc594cd 100644 --- a/CHANGE.md +++ b/CHANGE.md @@ -100,26 +100,6 @@ - [Fix bug in reference_config](https://github.com/apache/dubbo-go/pull/157) - [Fix high memory bug in zookeeper listener](https://github.com/apache/dubbo-go/pull/168) -### Enhancement - -- Split gettyRPCClient.close and gettyRPCClientPool.remove in protocol/dubbo/pool.go -- Remove client from pool before closing it -- Enhance the logic for fetching the local address -- Add protocol_conf default values -- Add task pool for getty -- Update getty: remove read queue -- Clean heartbeat from PendingResponse - -### Bugfixes - -- GettyRPCClientPool remove deadlock -- Fix failover cluster bug and url parameter retries change int to string type -- Fix url params unsafe map -- Read protocol config by map key in config yaml instead of protocol name -- Fix dubbo group issues #238/ -- Fix bug in reference_config -- Fix high memory bug in zookeeper listener - ## 1.1.0 ### New Features From b88cf66219ceda0b664c5fd18e022708d95fba20 Mon Sep 17 00:00:00 2001 From: alexstocks Date: Mon, 16 Mar 2020 11:03:56 +0800 Subject: [PATCH 49/61] Mod: split long line codes --- registry/kubernetes/registry_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/registry/kubernetes/registry_test.go b/registry/kubernetes/registry_test.go index 43a1bd79b6..d464c77df3 100644 --- a/registry/kubernetes/registry_test.go +++ b/registry/kubernetes/registry_test.go @@ -90,7 +90,9 @@ func (s *KubernetesRegistryTestSuite) TestConsumerDestroy() { r := s.initRegistry() - url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) + url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", + common.WithParamsValue(constant.CLUSTER_KEY, "mock"), + common.WithMethods([]string{"GetUser", "AddUser"})) listener, err := r.DoSubscribe(&url) if err != nil { @@ -111,7 +113,9 @@ func (s *KubernetesRegistryTestSuite) TestProviderDestroy() { r := s.initRegistry() - url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) + url, _ := common.NewURL("dubbo://127.0.0.1:20000/com.ikurento.user.UserProvider", + common.WithParamsValue(constant.CLUSTER_KEY, "mock"), + common.WithMethods([]string{"GetUser", "AddUser"})) err := r.Register(url) assert.NoError(t, err) @@ -126,7 +130,8 @@ func (s *KubernetesRegistryTestSuite) TestNewRegistry() { t := s.T() - regUrl, err := common.NewURL("registry://127.0.0.1:443", common.WithParamsValue(constant.ROLE_KEY, strconv.Itoa(common.PROVIDER))) + regUrl, err := common.NewURL("registry://127.0.0.1:443", + common.WithParamsValue(constant.ROLE_KEY, strconv.Itoa(common.PROVIDER))) if err != nil { t.Fatal(err) } From 35624ed557db354be2f751581dadeae137b6354d Mon Sep 17 00:00:00 2001 From: alexstocks Date: Mon, 16 Mar 2020 11:15:09 +0800 Subject: [PATCH 50/61] Imp: set the init len for a map --- remoting/kubernetes/listener.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remoting/kubernetes/listener.go b/remoting/kubernetes/listener.go index 87deeb38c9..4c198c66cc 100644 --- a/remoting/kubernetes/listener.go +++ b/remoting/kubernetes/listener.go @@ -40,7 +40,7 @@ type EventListener struct { func NewEventListener(client *Client) *EventListener { return &EventListener{ client: client, - keyMap: make(map[string]struct{}), + keyMap: make(map[string]struct{}, 8), } } From eeaa81778f3abb1373812459368257a2decaa6cb Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 16 Mar 2020 11:56:19 +0800 Subject: [PATCH 51/61] Fix latest comment --- registry/kubernetes/registry.go | 15 ++++++++------- remoting/kubernetes/client_test.go | 12 ++++++++---- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index 289a42aa5a..f8618caacc 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -89,11 +89,12 @@ func (r *kubernetesRegistry) CloseAndNilClient() { func (r *kubernetesRegistry) CloseListener() { r.cltLock.Lock() - if r.configListener != nil { - r.configListener.Close() + l := r.configListener + r.cltLock.Unlock() + if l != nil { + l.Close() } r.configListener = nil - r.cltLock.Unlock() } func (r *kubernetesRegistry) CreatePath(k string) error { @@ -124,11 +125,11 @@ func (r *kubernetesRegistry) DoSubscribe(svc *common.URL) (registry.Listener, er return nil, perrors.New("kubernetes client broken") } - // new client & listener - listener := kubernetes.NewEventListener(r.client) - r.listenerLock.Lock() - r.listener = listener + if r.listener == nil { + // double check + r.listener = kubernetes.NewEventListener(r.client) + } r.listenerLock.Unlock() } diff --git a/remoting/kubernetes/client_test.go b/remoting/kubernetes/client_test.go index d8ed782919..985aba6a94 100644 --- a/remoting/kubernetes/client_test.go +++ b/remoting/kubernetes/client_test.go @@ -328,14 +328,16 @@ func (s *KubernetesClientTestSuite) TestClientGetChildrenKVList() { wg := sync.WaitGroup{} wg.Add(1) + syncDataComplete := make(chan struct{}) + go func() { - defer wg.Done() wc, done, err := client.WatchWithPrefix(prefix) if err != nil { t.Fatal(err) } i := 0 + wg.Done() for { select { @@ -344,6 +346,7 @@ func (s *KubernetesClientTestSuite) TestClientGetChildrenKVList() { t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) if i == 3 { // already sync all event + syncDataComplete <- struct{}{} return } case <-done: @@ -353,6 +356,9 @@ func (s *KubernetesClientTestSuite) TestClientGetChildrenKVList() { } }() + // wait the watch goroutine start + wg.Wait() + expect := make(map[string]string) got := make(map[string]string) @@ -370,9 +376,7 @@ func (s *KubernetesClientTestSuite) TestClientGetChildrenKVList() { } } - // must wait client sync all create event - - wg.Wait() + <-syncDataComplete // start get all children kList, vList, err := client.GetChildren(prefix) From 5a6098856f4c4c5e758e248bf60a03bf8d28a651 Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 16 Mar 2020 14:11:51 +0800 Subject: [PATCH 52/61] Fix ci flow block issue --- registry/kubernetes/listener_test.go | 1 + registry/kubernetes/registry.go | 2 -- registry/kubernetes/registry_test.go | 9 ++------- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go index c9ff626088..f89154e32e 100644 --- a/registry/kubernetes/listener_test.go +++ b/registry/kubernetes/listener_test.go @@ -237,6 +237,7 @@ func (s *KubernetesRegistryTestSuite) SetupSuite() { if err := os.Setenv(nameSpaceKey, s.currentPod.GetNamespace()); err != nil { t.Fatal(err) } + } func (s *KubernetesRegistryTestSuite) TestDataChange() { diff --git a/registry/kubernetes/registry.go b/registry/kubernetes/registry.go index f8618caacc..7212a83d63 100644 --- a/registry/kubernetes/registry.go +++ b/registry/kubernetes/registry.go @@ -183,8 +183,6 @@ func newMockKubernetesRegistry( if err != nil { return nil, perrors.WithMessage(err, "new mock client") } - r.WaitGroup().Add(1) //zk client start successful, then wg +1 - go r.HandleClientRestart() r.InitListeners() return r, nil } diff --git a/registry/kubernetes/registry_test.go b/registry/kubernetes/registry_test.go index d464c77df3..10eed20526 100644 --- a/registry/kubernetes/registry_test.go +++ b/registry/kubernetes/registry_test.go @@ -50,7 +50,6 @@ func (s *KubernetesRegistryTestSuite) TestRegister() { if err != nil { t.Fatal(err) } - r.WaitGroup().Done() } func (s *KubernetesRegistryTestSuite) TestSubscribe() { @@ -80,8 +79,6 @@ func (s *KubernetesRegistryTestSuite) TestSubscribe() { } t.Logf("got event %s", serviceEvent) - - r.WaitGroup().Done() } func (s *KubernetesRegistryTestSuite) TestConsumerDestroy() { @@ -94,12 +91,12 @@ func (s *KubernetesRegistryTestSuite) TestConsumerDestroy() { common.WithParamsValue(constant.CLUSTER_KEY, "mock"), common.WithMethods([]string{"GetUser", "AddUser"})) - listener, err := r.DoSubscribe(&url) + _, err := r.DoSubscribe(&url) if err != nil { t.Fatal(err) } - listener.Close() + //listener.Close() time.Sleep(1e9) r.Destroy() @@ -119,8 +116,6 @@ func (s *KubernetesRegistryTestSuite) TestProviderDestroy() { err := r.Register(url) assert.NoError(t, err) - r.WaitGroup().Done() - time.Sleep(1e9) r.Destroy() assert.Equal(t, false, r.IsAvailable()) From 14ba4ff4acc602c01fef70c9047808dae4a8fc01 Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 16 Mar 2020 14:14:52 +0800 Subject: [PATCH 53/61] etcdv3 unit test, adapte for windows --- registry/etcdv3/listener_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/registry/etcdv3/listener_test.go b/registry/etcdv3/listener_test.go index 5f5d95aa36..e691ae3cf1 100644 --- a/registry/etcdv3/listener_test.go +++ b/registry/etcdv3/listener_test.go @@ -18,7 +18,7 @@ package etcdv3 import ( - "os/exec" + "os" "testing" "time" ) @@ -70,7 +70,7 @@ func (suite *RegistryTestSuite) SetupSuite() { func (suite *RegistryTestSuite) TearDownSuite() { suite.etcd.Close() // clean the etcd workdir - if err := exec.Command("rm", "-rf", defaultEtcdV3WorkDir).Run(); err != nil { + if err := os.RemoveAll(defaultEtcdV3WorkDir); err != nil { suite.FailNow(err.Error()) } } From ab7db8a849f9abf8180d6c658e55b259e4da0023 Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 16 Mar 2020 14:17:01 +0800 Subject: [PATCH 54/61] simple the client validate method --- remoting/kubernetes/client.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index 3072aee2cc..fbabd55bef 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -634,12 +634,8 @@ func (c *Client) Valid() bool { default: } c.lock.RLock() - if c.rawClient == nil { - c.lock.RUnlock() - return false - } - c.lock.RUnlock() - return true + defer c.lock.RUnlock() + return c.rawClient != nil } // Done From cef0d5dd8bd8d7e5b9fbdb974ff510f8705c8247 Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 16 Mar 2020 14:19:21 +0800 Subject: [PATCH 55/61] short test time-cost for watch set --- remoting/kubernetes/watch_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/remoting/kubernetes/watch_test.go b/remoting/kubernetes/watch_test.go index 10fbc7eb53..7bfc35ecfd 100644 --- a/remoting/kubernetes/watch_test.go +++ b/remoting/kubernetes/watch_test.go @@ -25,9 +25,9 @@ import ( "time" ) -func TestStore(t *testing.T) { +func TestWatchSet(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() s := newWatcherSet(ctx) From 938d5edc90e4c3b5a8ae24281a99040bac4c7977 Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 16 Mar 2020 14:36:58 +0800 Subject: [PATCH 56/61] Add more rich unit-test --- registry/kubernetes/listener_test.go | 4 ++++ registry/kubernetes/registry_test.go | 9 +++++++++ remoting/kubernetes/watch_test.go | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go index f89154e32e..88e1589020 100644 --- a/registry/kubernetes/listener_test.go +++ b/registry/kubernetes/listener_test.go @@ -19,6 +19,8 @@ package kubernetes import ( "encoding/json" + "net/http" + _ "net/http/pprof" "os" "strconv" "testing" @@ -238,6 +240,8 @@ func (s *KubernetesRegistryTestSuite) SetupSuite() { t.Fatal(err) } + go http.ListenAndServe(":6061", nil) + } func (s *KubernetesRegistryTestSuite) TestDataChange() { diff --git a/registry/kubernetes/registry_test.go b/registry/kubernetes/registry_test.go index 10eed20526..b9f817b8de 100644 --- a/registry/kubernetes/registry_test.go +++ b/registry/kubernetes/registry_test.go @@ -135,3 +135,12 @@ func (s *KubernetesRegistryTestSuite) TestNewRegistry() { t.Fatal("not in cluster, should be a err") } } + +func (s *KubernetesRegistryTestSuite) TestHandleClientRestart() { + + r := s.initRegistry() + r.WaitGroup().Add(1) + go r.HandleClientRestart() + time.Sleep(timeSecondDuration(1)) + r.CloseAndNilClient() +} diff --git a/remoting/kubernetes/watch_test.go b/remoting/kubernetes/watch_test.go index 7bfc35ecfd..8889103be2 100644 --- a/remoting/kubernetes/watch_test.go +++ b/remoting/kubernetes/watch_test.go @@ -27,7 +27,7 @@ import ( func TestWatchSet(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() s := newWatcherSet(ctx) From 52a9e288ab160f6357e2ff2df0222d76b32669ae Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 16 Mar 2020 14:42:19 +0800 Subject: [PATCH 57/61] etcdv3 unit test, adapte for windows --- remoting/etcdv3/client_test.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/remoting/etcdv3/client_test.go b/remoting/etcdv3/client_test.go index 692ae2fb66..895cc2954a 100644 --- a/remoting/etcdv3/client_test.go +++ b/remoting/etcdv3/client_test.go @@ -18,9 +18,8 @@ package etcdv3 import ( - "fmt" "net/url" - "os/exec" + "os" "path" "reflect" "strings" @@ -115,7 +114,7 @@ func (suite *ClientTestSuite) SetupSuite() { // stop etcd server func (suite *ClientTestSuite) TearDownSuite() { suite.etcd.Close() - if err := exec.Command("rm", "-rf", defaultEtcdV3WorkDir).Run(); err != nil { + if err := os.RemoveAll(defaultEtcdV3WorkDir); err != nil { suite.FailNow(err.Error()) } } @@ -141,8 +140,6 @@ func (suite *ClientTestSuite) SetupTest() { func (suite *ClientTestSuite) TestClientClose() { - fmt.Println("called client close") - c := suite.client t := suite.T() @@ -154,8 +151,6 @@ func (suite *ClientTestSuite) TestClientClose() { func (suite *ClientTestSuite) TestClientValid() { - fmt.Println("called client valid") - c := suite.client t := suite.T() From 0769966f67ff27be3e3065c242f7ab88c60a18c0 Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 16 Mar 2020 14:44:37 +0800 Subject: [PATCH 58/61] Fix ut nil pointer issue --- registry/kubernetes/registry_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registry/kubernetes/registry_test.go b/registry/kubernetes/registry_test.go index b9f817b8de..25cc777c4d 100644 --- a/registry/kubernetes/registry_test.go +++ b/registry/kubernetes/registry_test.go @@ -142,5 +142,5 @@ func (s *KubernetesRegistryTestSuite) TestHandleClientRestart() { r.WaitGroup().Add(1) go r.HandleClientRestart() time.Sleep(timeSecondDuration(1)) - r.CloseAndNilClient() + r.client.Close() } From 2eeb73299b8354680716975bf8857f2157fc443d Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 16 Mar 2020 15:04:36 +0800 Subject: [PATCH 59/61] Fix kubernetes registry ut block issue --- registry/kubernetes/listener_test.go | 4 ---- registry/kubernetes/registry_test.go | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go index 88e1589020..f89154e32e 100644 --- a/registry/kubernetes/listener_test.go +++ b/registry/kubernetes/listener_test.go @@ -19,8 +19,6 @@ package kubernetes import ( "encoding/json" - "net/http" - _ "net/http/pprof" "os" "strconv" "testing" @@ -240,8 +238,6 @@ func (s *KubernetesRegistryTestSuite) SetupSuite() { t.Fatal(err) } - go http.ListenAndServe(":6061", nil) - } func (s *KubernetesRegistryTestSuite) TestDataChange() { diff --git a/registry/kubernetes/registry_test.go b/registry/kubernetes/registry_test.go index 25cc777c4d..ea6d7663a9 100644 --- a/registry/kubernetes/registry_test.go +++ b/registry/kubernetes/registry_test.go @@ -65,6 +65,7 @@ func (s *KubernetesRegistryTestSuite) TestSubscribe() { if err != nil { t.Fatal(err) } + time.Sleep(1e9) go func() { err := r.Register(url) From 164bf89e05626b2374981dc67ec047783d7f362f Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 16 Mar 2020 17:57:31 +0800 Subject: [PATCH 60/61] Fix kubernetes registry ut block issue --- remoting/kubernetes/client.go | 14 ++++++++++---- remoting/kubernetes/client_test.go | 14 ++++++++++++-- remoting/kubernetes/watch.go | 2 ++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/remoting/kubernetes/client.go b/remoting/kubernetes/client.go index fbabd55bef..0c9ffd2b91 100644 --- a/remoting/kubernetes/client.go +++ b/remoting/kubernetes/client.go @@ -152,6 +152,8 @@ func newMockClient(namespace string, mockClientGenerator func() (kubernetes.Inte return nil, perrors.WithMessage(err, "init watcherSet") } + c.lastResourceVersion = c.currentPod.GetResourceVersion() + // start kubernetes watch loop if err := c.watchPods(); err != nil { return nil, perrors.WithMessage(err, "watch pods") @@ -324,7 +326,7 @@ func (c *Client) watchPodsLoop() { select { // double check ctx case <-c.ctx.Done(): - logger.Info("the kubernetes client stopped, resultChan len %d", len(wc.ResultChan())) + logger.Infof("the kubernetes client stopped, resultChan len %d", len(wc.ResultChan())) return // get one element from result-chan @@ -358,6 +360,7 @@ func (c *Client) watchPodsLoop() { continue } + logger.Debugf("kubernetes got pod %#v", p) // handle the watched pod go c.handleWatchedPodEvent(p, event.Type) } @@ -456,10 +459,14 @@ func (c *Client) readCurrentPod() (*v1.Pod, error) { // create k/v pair in watcher-set func (c *Client) Create(k, v string) error { + // the read current pod must be lock, protect every + // create operation can be atomic + c.lock.Lock() + defer c.lock.Unlock() + // 1. accord old pod && (k, v) assemble new pod dubbo annotion v // 2. get patch data // 3. PATCH the pod - currentPod, err := c.readCurrentPod() if err != nil { return perrors.WithMessage(err, "read current pod") @@ -480,9 +487,8 @@ func (c *Client) Create(k, v string) error { return perrors.WithMessage(err, "patch current pod") } - c.lock.Lock() c.currentPod = updatedPod - c.lock.Unlock() + logger.Debugf("put the @key = %s @value = %s success", k, v) // not update the watcherSet, the watcherSet should be write by the watchPodsLoop return nil } diff --git a/remoting/kubernetes/client_test.go b/remoting/kubernetes/client_test.go index 985aba6a94..342285b345 100644 --- a/remoting/kubernetes/client_test.go +++ b/remoting/kubernetes/client_test.go @@ -19,7 +19,10 @@ package kubernetes import ( "encoding/json" + "fmt" + "net/http" "os" + "runtime" "strings" "sync" "testing" @@ -216,11 +219,15 @@ func (s *KubernetesClientTestSuite) initClient() *Client { if err != nil { t.Fatal(err) } + + time.Sleep(time.Second) return client } func (s *KubernetesClientTestSuite) SetupSuite() { + runtime.GOMAXPROCS(1) + t := s.T() // 1. install test data @@ -236,6 +243,8 @@ func (s *KubernetesClientTestSuite) SetupSuite() { t.Fatal(err) } + go http.ListenAndServe(":6061", nil) + } func (s *KubernetesClientTestSuite) TestReadCurrentPodName() { @@ -336,14 +345,15 @@ func (s *KubernetesClientTestSuite) TestClientGetChildrenKVList() { if err != nil { t.Fatal(err) } - i := 0 + wg.Done() + i := 0 for { select { case e := <-wc: i++ - t.Logf("got event %v k %s v %s", e.EventType, e.Key, e.Value) + fmt.Printf("got event %v k %s v %s\n", e.EventType, e.Key, e.Value) if i == 3 { // already sync all event syncDataComplete <- struct{}{} diff --git a/remoting/kubernetes/watch.go b/remoting/kubernetes/watch.go index 835bcdb9b7..c99a3ebcc0 100644 --- a/remoting/kubernetes/watch.go +++ b/remoting/kubernetes/watch.go @@ -186,6 +186,8 @@ func (s *watcherSetImpl) Put(watcherEvent *WatcherEvent) error { // notify watcher for _, w := range s.watchers { + w := w + if !strings.Contains(watcherEvent.Key, w.interested.key) { // this watcher no interest in this element continue From e48b6989fd9198527662cb7cac1fedf8a80096c5 Mon Sep 17 00:00:00 2001 From: "scott.wang" Date: Mon, 16 Mar 2020 18:05:24 +0800 Subject: [PATCH 61/61] Add new registry block time, wait the watch groutine start --- registry/kubernetes/listener_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/registry/kubernetes/listener_test.go b/registry/kubernetes/listener_test.go index f89154e32e..c50b5b670a 100644 --- a/registry/kubernetes/listener_test.go +++ b/registry/kubernetes/listener_test.go @@ -22,6 +22,7 @@ import ( "os" "strconv" "testing" + "time" ) import ( @@ -212,6 +213,7 @@ func (s *KubernetesRegistryTestSuite) initRegistry() *kubernetesRegistry { t.Fatal(err) } + time.Sleep(time.Second) return mock.(*kubernetesRegistry) }