From ae9c7d419ae755c15fa1c149270c87e399a06407 Mon Sep 17 00:00:00 2001 From: flixgithub Date: Sat, 8 Feb 2020 00:30:51 +0800 Subject: [PATCH 01/22] grpc client and server trace function --- protocol/grpc/client.go | 26 +++++++++++++++++++++++--- protocol/grpc/server.go | 16 +++++++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/protocol/grpc/client.go b/protocol/grpc/client.go index d35a2c770c..377fae009c 100644 --- a/protocol/grpc/client.go +++ b/protocol/grpc/client.go @@ -18,9 +18,14 @@ limitations under the License. package grpc import ( + "github.com/opentracing/opentracing-go" "reflect" ) +import ( + "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" +) + import ( "google.golang.org/grpc" ) @@ -39,9 +44,24 @@ type Client struct { // NewClient ... func NewClient(url common.URL) *Client { - conn, err := grpc.Dial(url.Location, grpc.WithInsecure(), grpc.WithBlock()) - if err != nil { - panic(err) + var ( + conn *grpc.ClientConn + err error + tracer opentracing.Tracer + ) + // if global trace instance was set in filter, it means trace function enabled + if opentracing.IsGlobalTracerRegistered() { + tracer = opentracing.GlobalTracer() + conn, err = grpc.Dial(url.Location, grpc.WithInsecure(), grpc.WithBlock(), + grpc.WithUnaryInterceptor(otgrpc.OpenTracingClientInterceptor(tracer))) + if err != nil { + panic(err) + } + } else { + conn, err = grpc.Dial(url.Location, grpc.WithInsecure(), grpc.WithBlock()) + if err != nil { + panic(err) + } } key := url.GetParam(constant.BEAN_NAME_KEY, "") diff --git a/protocol/grpc/server.go b/protocol/grpc/server.go index 19b9db4ac7..fcf504838b 100644 --- a/protocol/grpc/server.go +++ b/protocol/grpc/server.go @@ -19,10 +19,15 @@ package grpc import ( "fmt" + "github.com/opentracing/opentracing-go" "net" "reflect" ) +import ( + "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" +) + import ( "google.golang.org/grpc" ) @@ -57,13 +62,22 @@ func (s *Server) Start(url common.URL) { var ( addr string err error + server *grpc.Server ) addr = url.Location lis, err := net.Listen("tcp", addr) if err != nil { panic(err) } - server := grpc.NewServer() + + // if global trace instance was set, then server tracer instance can be get, and span context can also be get + if opentracing.IsGlobalTracerRegistered() { + tracer := opentracing.GlobalTracer() + server = grpc.NewServer(grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(tracer))) + } else { + server = grpc.NewServer() + } + key := url.GetParam(constant.BEAN_NAME_KEY, "") service := config.GetProviderService(key) From 0b60bb859568622e9b2f0e2c8c95ccd041842d83 Mon Sep 17 00:00:00 2001 From: flixgithub Date: Sat, 8 Feb 2020 11:06:45 +0800 Subject: [PATCH 02/22] go fmt --- protocol/grpc/client.go | 6 +++--- protocol/grpc/server.go | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/protocol/grpc/client.go b/protocol/grpc/client.go index 377fae009c..74cd1c6aaf 100644 --- a/protocol/grpc/client.go +++ b/protocol/grpc/client.go @@ -45,11 +45,11 @@ type Client struct { // NewClient ... func NewClient(url common.URL) *Client { var ( - conn *grpc.ClientConn - err error + conn *grpc.ClientConn + err error tracer opentracing.Tracer ) - // if global trace instance was set in filter, it means trace function enabled + // if global trace instance was set in filter, it means trace function enabled if opentracing.IsGlobalTracerRegistered() { tracer = opentracing.GlobalTracer() conn, err = grpc.Dial(url.Location, grpc.WithInsecure(), grpc.WithBlock(), diff --git a/protocol/grpc/server.go b/protocol/grpc/server.go index fcf504838b..8491141aa4 100644 --- a/protocol/grpc/server.go +++ b/protocol/grpc/server.go @@ -60,8 +60,8 @@ type DubboGrpcService interface { // Start ... func (s *Server) Start(url common.URL) { var ( - addr string - err error + addr string + err error server *grpc.Server ) addr = url.Location @@ -78,7 +78,6 @@ func (s *Server) Start(url common.URL) { server = grpc.NewServer() } - key := url.GetParam(constant.BEAN_NAME_KEY, "") service := config.GetProviderService(key) From a4986f53f55e5cdf8836f44b55fdf13db72ad2b5 Mon Sep 17 00:00:00 2001 From: flixgithub Date: Sat, 8 Feb 2020 11:42:03 +0800 Subject: [PATCH 03/22] go fmt --- go.mod | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/go.mod b/go.mod index 54b39d322e..44eb8c4b99 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.9.5 // indirect + github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 github.com/hashicorp/consul v1.5.3 github.com/hashicorp/consul/api v1.1.0 github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 // indirect diff --git a/go.sum b/go.sum index 0461f29653..b0799377a7 100644 --- a/go.sum +++ b/go.sum @@ -198,6 +198,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU= +github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/consul v1.5.3 h1:EmTWRf/cuqZk6Ug9tgFUVE9xNgJPpmBvJwJMvm+agSk= From fd77a7a4deb7fc1aff8bec149840ad5bc9b7f18d Mon Sep 17 00:00:00 2001 From: flixgithub Date: Mon, 10 Feb 2020 11:27:17 +0800 Subject: [PATCH 04/22] split package --- protocol/grpc/client.go | 2 +- protocol/grpc/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/protocol/grpc/client.go b/protocol/grpc/client.go index 74cd1c6aaf..038bf93118 100644 --- a/protocol/grpc/client.go +++ b/protocol/grpc/client.go @@ -18,12 +18,12 @@ limitations under the License. package grpc import ( - "github.com/opentracing/opentracing-go" "reflect" ) import ( "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" + "github.com/opentracing/opentracing-go" ) import ( diff --git a/protocol/grpc/server.go b/protocol/grpc/server.go index 8491141aa4..83dab37ed2 100644 --- a/protocol/grpc/server.go +++ b/protocol/grpc/server.go @@ -19,13 +19,13 @@ package grpc import ( "fmt" - "github.com/opentracing/opentracing-go" "net" "reflect" ) import ( "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" + "github.com/opentracing/opentracing-go" ) import ( From e31ac1dfd3bc0732bd88f139564130bbce8754ad Mon Sep 17 00:00:00 2001 From: alexstocks Date: Tue, 10 Mar 2020 19:49:32 +0800 Subject: [PATCH 05/22] Add: go report card --- README.md | 2 ++ README_CN.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index 1dde951d35..2975ce9de0 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ [![Build Status](https://travis-ci.org/apache/dubbo-go.svg?branch=master)](https://travis-ci.org/apache/dubbo-go) [![codecov](https://codecov.io/gh/apache/dubbo-go/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/dubbo-go) [![go.dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/apache/dubbo-go?tab=doc) +[![Go Report Card](https://goreportcard.com/badge/github.com/apache/dubbo-go)](https://goreportcard.com/report/github.com/apache/dubbo-go) +![license](https://img.shields.io/badge/license-Apache--2.0-green.svg) --- Apache Dubbo Go Implementation. diff --git a/README_CN.md b/README_CN.md index ade924e7a9..62c5b06575 100644 --- a/README_CN.md +++ b/README_CN.md @@ -3,6 +3,8 @@ [![Build Status](https://travis-ci.org/apache/dubbo-go.svg?branch=master)](https://travis-ci.org/apache/dubbo-go) [![codecov](https://codecov.io/gh/apache/dubbo-go/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/dubbo-go) [![go.dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/apache/dubbo-go?tab=doc) +[![Go Report Card](https://goreportcard.com/badge/github.com/apache/dubbo-go)](https://goreportcard.com/report/github.com/apache/dubbo-go) +![license](https://img.shields.io/badge/license-Apache--2.0-green.svg) --- Apache Dubbo Go 语言实现 From f85d07bc8ca5ea94837ff11c305cdee7c898d486 Mon Sep 17 00:00:00 2001 From: Joe Zou Date: Thu, 12 Mar 2020 00:46:26 +0800 Subject: [PATCH 06/22] mod grpc tracing --- protocol/grpc/client.go | 5 ++++- protocol/grpc/server.go | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/protocol/grpc/client.go b/protocol/grpc/client.go index 038bf93118..0f7651ed24 100644 --- a/protocol/grpc/client.go +++ b/protocol/grpc/client.go @@ -53,7 +53,10 @@ func NewClient(url common.URL) *Client { if opentracing.IsGlobalTracerRegistered() { tracer = opentracing.GlobalTracer() conn, err = grpc.Dial(url.Location, grpc.WithInsecure(), grpc.WithBlock(), - grpc.WithUnaryInterceptor(otgrpc.OpenTracingClientInterceptor(tracer))) + grpc.WithUnaryInterceptor( + otgrpc.OpenTracingClientInterceptor(tracer, otgrpc.LogPayloads())), + grpc.WithStreamInterceptor( + otgrpc.OpenTracingStreamClientInterceptor(tracer, otgrpc.LogPayloads()))) if err != nil { panic(err) } diff --git a/protocol/grpc/server.go b/protocol/grpc/server.go index 83dab37ed2..5574388f77 100644 --- a/protocol/grpc/server.go +++ b/protocol/grpc/server.go @@ -73,7 +73,9 @@ func (s *Server) Start(url common.URL) { // if global trace instance was set, then server tracer instance can be get, and span context can also be get if opentracing.IsGlobalTracerRegistered() { tracer := opentracing.GlobalTracer() - server = grpc.NewServer(grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(tracer))) + server = grpc.NewServer( + grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(tracer)), + grpc.StreamInterceptor(otgrpc.OpenTracingStreamServerInterceptor(tracer))) } else { server = grpc.NewServer() } From d99e9b32cc7ed4510952095ac57b945607246407 Mon Sep 17 00:00:00 2001 From: pantianying <601666418@qq.com> Date: Thu, 12 Mar 2020 12:40:35 +0800 Subject: [PATCH 07/22] user can add attachment --- common/proxy/proxy.go | 8 ++++++++ config/generic_service.go | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/common/proxy/proxy.go b/common/proxy/proxy.go index 6765a810a5..68ba3ff788 100644 --- a/common/proxy/proxy.go +++ b/common/proxy/proxy.go @@ -140,6 +140,14 @@ func (p *Proxy) Implement(v common.RPCService) { inv.SetAttachments(k, value) } + // add user setAttachment + atm := invCtx.Value("attachment") + if m, ok := atm.(map[string]string); ok { + for k, value := range m { + inv.SetAttachments(k, value) + } + } + result := p.invoke.Invoke(invCtx, inv) err = result.Error() diff --git a/config/generic_service.go b/config/generic_service.go index 9895486e97..b66e399f9e 100644 --- a/config/generic_service.go +++ b/config/generic_service.go @@ -17,9 +17,11 @@ package config +import "context" + // GenericService ... type GenericService struct { - Invoke func(req []interface{}) (interface{}, error) `dubbo:"$invoke"` + Invoke func(ctx context.Context, req []interface{}) (interface{}, error) `dubbo:"$invoke"` referenceStr string } From 3606bc1f4269c5f2930551ed324b891d2ff67b21 Mon Sep 17 00:00:00 2001 From: Joe Zou Date: Thu, 12 Mar 2020 22:48:03 +0800 Subject: [PATCH 08/22] mod grpc tracing --- protocol/grpc/client.go | 25 +++++++++---------------- protocol/grpc/server.go | 14 +++++--------- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/protocol/grpc/client.go b/protocol/grpc/client.go index 0f7651ed24..18781e1422 100644 --- a/protocol/grpc/client.go +++ b/protocol/grpc/client.go @@ -49,22 +49,15 @@ func NewClient(url common.URL) *Client { err error tracer opentracing.Tracer ) - // if global trace instance was set in filter, it means trace function enabled - if opentracing.IsGlobalTracerRegistered() { - tracer = opentracing.GlobalTracer() - conn, err = grpc.Dial(url.Location, grpc.WithInsecure(), grpc.WithBlock(), - grpc.WithUnaryInterceptor( - otgrpc.OpenTracingClientInterceptor(tracer, otgrpc.LogPayloads())), - grpc.WithStreamInterceptor( - otgrpc.OpenTracingStreamClientInterceptor(tracer, otgrpc.LogPayloads()))) - if err != nil { - panic(err) - } - } else { - conn, err = grpc.Dial(url.Location, grpc.WithInsecure(), grpc.WithBlock()) - if err != nil { - panic(err) - } + // if global trace instance was set , it means trace function enabled. If not , will return Nooptracer + tracer = opentracing.GlobalTracer() + conn, err = grpc.Dial(url.Location, grpc.WithInsecure(), grpc.WithBlock(), + grpc.WithUnaryInterceptor( + otgrpc.OpenTracingClientInterceptor(tracer, otgrpc.LogPayloads())), + grpc.WithStreamInterceptor( + otgrpc.OpenTracingStreamClientInterceptor(tracer, otgrpc.LogPayloads()))) + if err != nil { + panic(err) } key := url.GetParam(constant.BEAN_NAME_KEY, "") diff --git a/protocol/grpc/server.go b/protocol/grpc/server.go index 5574388f77..4114449f6d 100644 --- a/protocol/grpc/server.go +++ b/protocol/grpc/server.go @@ -70,15 +70,11 @@ func (s *Server) Start(url common.URL) { panic(err) } - // if global trace instance was set, then server tracer instance can be get, and span context can also be get - if opentracing.IsGlobalTracerRegistered() { - tracer := opentracing.GlobalTracer() - server = grpc.NewServer( - grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(tracer)), - grpc.StreamInterceptor(otgrpc.OpenTracingStreamServerInterceptor(tracer))) - } else { - server = grpc.NewServer() - } + // if global trace instance was set , then server tracer instance can be get. If not , will return Nooptracer + tracer := opentracing.GlobalTracer() + server = grpc.NewServer( + grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(tracer)), + grpc.StreamInterceptor(otgrpc.OpenTracingStreamServerInterceptor(tracer))) key := url.GetParam(constant.BEAN_NAME_KEY, "") service := config.GetProviderService(key) From 5f2981f748b89af8cf0f5f3dd42d4dbc7842f845 Mon Sep 17 00:00:00 2001 From: Joe Zou Date: Thu, 12 Mar 2020 22:53:06 +0800 Subject: [PATCH 09/22] mod grpc tracing --- protocol/grpc/client.go | 4 +--- protocol/grpc/server.go | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/protocol/grpc/client.go b/protocol/grpc/client.go index 18781e1422..54983fbd65 100644 --- a/protocol/grpc/client.go +++ b/protocol/grpc/client.go @@ -53,9 +53,7 @@ func NewClient(url common.URL) *Client { tracer = opentracing.GlobalTracer() conn, err = grpc.Dial(url.Location, grpc.WithInsecure(), grpc.WithBlock(), grpc.WithUnaryInterceptor( - otgrpc.OpenTracingClientInterceptor(tracer, otgrpc.LogPayloads())), - grpc.WithStreamInterceptor( - otgrpc.OpenTracingStreamClientInterceptor(tracer, otgrpc.LogPayloads()))) + otgrpc.OpenTracingClientInterceptor(tracer, otgrpc.LogPayloads()))) if err != nil { panic(err) } diff --git a/protocol/grpc/server.go b/protocol/grpc/server.go index 4114449f6d..203d107c66 100644 --- a/protocol/grpc/server.go +++ b/protocol/grpc/server.go @@ -73,8 +73,7 @@ func (s *Server) Start(url common.URL) { // if global trace instance was set , then server tracer instance can be get. If not , will return Nooptracer tracer := opentracing.GlobalTracer() server = grpc.NewServer( - grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(tracer)), - grpc.StreamInterceptor(otgrpc.OpenTracingStreamServerInterceptor(tracer))) + grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(tracer))) key := url.GetParam(constant.BEAN_NAME_KEY, "") service := config.GetProviderService(key) From 59c11c0081a0f0504523cb2b61806798c0d000e3 Mon Sep 17 00:00:00 2001 From: Joe Zou Date: Thu, 12 Mar 2020 22:55:57 +0800 Subject: [PATCH 10/22] mod grpc tracing --- protocol/grpc/server.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/protocol/grpc/server.go b/protocol/grpc/server.go index 203d107c66..b25125a06c 100644 --- a/protocol/grpc/server.go +++ b/protocol/grpc/server.go @@ -62,7 +62,6 @@ func (s *Server) Start(url common.URL) { var ( addr string err error - server *grpc.Server ) addr = url.Location lis, err := net.Listen("tcp", addr) @@ -72,7 +71,7 @@ func (s *Server) Start(url common.URL) { // if global trace instance was set , then server tracer instance can be get. If not , will return Nooptracer tracer := opentracing.GlobalTracer() - server = grpc.NewServer( + server := grpc.NewServer( grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(tracer))) key := url.GetParam(constant.BEAN_NAME_KEY, "") From 7db3a2a7a043cdbb95f2fa5dc955698d2a0a9e79 Mon Sep 17 00:00:00 2001 From: Joe Zou Date: Thu, 12 Mar 2020 22:56:37 +0800 Subject: [PATCH 11/22] mod grpc tracing --- protocol/grpc/client.go | 9 ++------- protocol/grpc/server.go | 4 ++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/protocol/grpc/client.go b/protocol/grpc/client.go index 54983fbd65..0738dc75c9 100644 --- a/protocol/grpc/client.go +++ b/protocol/grpc/client.go @@ -44,14 +44,9 @@ type Client struct { // NewClient ... func NewClient(url common.URL) *Client { - var ( - conn *grpc.ClientConn - err error - tracer opentracing.Tracer - ) // if global trace instance was set , it means trace function enabled. If not , will return Nooptracer - tracer = opentracing.GlobalTracer() - conn, err = grpc.Dial(url.Location, grpc.WithInsecure(), grpc.WithBlock(), + tracer := opentracing.GlobalTracer() + conn, err := grpc.Dial(url.Location, grpc.WithInsecure(), grpc.WithBlock(), grpc.WithUnaryInterceptor( otgrpc.OpenTracingClientInterceptor(tracer, otgrpc.LogPayloads()))) if err != nil { diff --git a/protocol/grpc/server.go b/protocol/grpc/server.go index b25125a06c..3e0ee64ca8 100644 --- a/protocol/grpc/server.go +++ b/protocol/grpc/server.go @@ -60,8 +60,8 @@ type DubboGrpcService interface { // Start ... func (s *Server) Start(url common.URL) { var ( - addr string - err error + addr string + err error ) addr = url.Location lis, err := net.Listen("tcp", addr) From be26d4160626d24329a29812d0a87d35f52e398d Mon Sep 17 00:00:00 2001 From: Joe Zou Date: Fri, 13 Mar 2020 20:59:04 +0800 Subject: [PATCH 12/22] merge import package --- protocol/grpc/client.go | 3 --- protocol/grpc/server.go | 3 --- 2 files changed, 6 deletions(-) diff --git a/protocol/grpc/client.go b/protocol/grpc/client.go index 0738dc75c9..6026f0991b 100644 --- a/protocol/grpc/client.go +++ b/protocol/grpc/client.go @@ -24,9 +24,6 @@ import ( import ( "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" "github.com/opentracing/opentracing-go" -) - -import ( "google.golang.org/grpc" ) diff --git a/protocol/grpc/server.go b/protocol/grpc/server.go index 3e0ee64ca8..cc184bf3cf 100644 --- a/protocol/grpc/server.go +++ b/protocol/grpc/server.go @@ -26,9 +26,6 @@ import ( import ( "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" "github.com/opentracing/opentracing-go" -) - -import ( "google.golang.org/grpc" ) From 30303276df82a380747b72596ee3af5c2a3d927d Mon Sep 17 00:00:00 2001 From: alexstocks Date: Sat, 14 Mar 2020 13:42:39 +0800 Subject: [PATCH 13/22] update hessian2 version --- .../apache-release-procedure-20200306.md | 448 ++++++++++++++++++ doc/apache/release_note.md | 11 + go.mod | 2 +- go.sum | 4 +- 4 files changed, 462 insertions(+), 3 deletions(-) create mode 100644 doc/apache/apache-release-procedure-20200306.md create mode 100644 doc/apache/release_note.md diff --git a/doc/apache/apache-release-procedure-20200306.md b/doc/apache/apache-release-procedure-20200306.md new file mode 100644 index 0000000000..3f677ff56b --- /dev/null +++ b/doc/apache/apache-release-procedure-20200306.md @@ -0,0 +1,448 @@ + +# Apache 软件发版流程 + +> author: wongoo@apache.org +> last updated: 2020-03-06 + +Apache开源软件是有社区驱动的，为了提高发布软件质量而指定了软件发布流程，本文主要介绍此流程，以给第一次发布打包的apacher参考。 + +如果你要准备打包一个apache软件了，想必你已经是一个项目的committer了，而且知道社区、PMC这些概念，而你现在还担任本次发布的 release manager 一职。 + +发版流程其实也很简单，无非如下： +1. 整理变更内容，打包并对打包文件签名； +2. 将签名文件上传apache svn仓库； +3. 发邮件请社区PMC大佬投票； +4. 投票通过后发一个投票结果通告邮件； +5. 发版 +6. 发版邮件通告社区新版本发布； + +下面详细整理发版的一些流程步骤，使用 dubbo 的子项目 dubbog-go-hessian2 发版为例！ + + +## 1. 发版准备 + +发版文件需要签名，需要安装pgp工具. + +```bash +$ brew install gpg +$ gpg --version +$ gpg --full-gen-key + (1) RSA and RSA (default) <-- RSA 类型 + What keysize do you want? (2048) 4096 <-- key大小为4096 + 0 = key does not expire <-- 永不过期 + Real name: Liu Yang + Email address: wongoo@apache.org + Comment: CODE SIGNING KEY + + gpg: /Users/gelnyang/.gnupg/trustdb.gpg: trustdb created + gpg: key 7DB68550D366E4C0 marked as ultimately trusted + gpg: revocation certificate stored as '/Users/gelnyang/.gnupg/openpgp-revocs.d/1376A2FF67E4C477573909BD7DB68550D366E4C0.rev' + public and secret key created and signed. + + pub rsa4096 2019-10-17 [SC] + 1376A2FF67E4C477573909BD7DB68550D366E4C0 + uid Liu Yang (CODE SIGNING KEY) + sub rsa4096 2019-10-17 [E] + +$ gpg --list-keys + gpg: checking the trustdb + gpg: marginals needed: 3 completes needed: 1 trust model: pgp + gpg: depth: 0 valid: 1 signed: 0 trust: 0-, 0q, 0n, 0m, 0f, 1u + /Users/gelnyang/.gnupg/pubring.kbx + ---------------------------------- + pub rsa4096 2019-10-17 [SC] + 1376A2FF67E4C477573909BD7DB68550D366E4C0 + uid [ultimate] Liu Yang (CODE SIGNING KEY) + sub rsa4096 2019-10-17 [E] + + +# 公钥服务器是网络上专门储存用户公钥的服务器 +# 通过key id发送public key到keyserver +$ gpg --keyserver pgpkeys.mit.edu --send-key 1376A2FF67E4C477573909BD7DB68550D366E4C0 + gpg: sending key 7DB68550D366E4C0 to hkp://pgpkeys.mit.edu +# 其中，pgpkeys.mit.edu为随意挑选的keyserver，keyserver列表为：https://sks-keyservers.net/status/，为相互之间是自动同步的，选任意一个都可以。 + +# 如果有多个public key，设置默认key。修改 ~/.gnupg/gpg.conf +$ vi ~/.gnupg/gpg.conf +default-key 7DB68550D366E4C0 + +# 如果有多个public key, 也可以删除无用的key： +### 先删除私钥，再删除公钥 +$ gpg --yes --delete-secret-keys shenglicao2@gmail.com ###老的私钥，指明邮箱即可 +$ gpg --delete-keys 1808C6444C781C0AEA0AAD4C4D6A8007D20DB8A4 + +## 由于公钥服务器没有检查机制，任何人都可以用你的名义上传公钥，所以没有办法保证服务器上的公钥的可靠性。 +## 通常，你可以在网站上公布一个公钥指纹，让其他人核对下载到的公钥是否为真。 +# fingerprint参数生成公钥指纹： +$ gpg --fingerprint wongoo + + pub rsa4096 2019-10-17 [SC] + 1376 A2FF 67E4 C477 5739 09BD 7DB6 8550 D366 E4C0 + uid [ultimate] Liu Yang (CODE SIGNING KEY) + sub rsa4096 2019-10-17 [E] + # 将上面的 fingerprint （即 1376 A2FF 67E4 C477 5739 09BD 7DB6 8550 D366 E4C0）粘贴到自己的用户信息中： + # https://id.apache.org OpenPGP Public Key Primary Fingerprint: +``` + +> 详细参考： +> - 发布签名: http://www.apache.org/dev/release-signing.html +> - 发布策略: http://www.apache.org/dev/release-distribution +> - 将密钥上传到公共密钥服务器: https://www.apache.org/dev/openpgp.html#generate-key + +## 2. 打包签名 + +准备打包前（尤其提第一次打包）需要注意以下内容: +- 每个文件的LICENSE头部是否正确, 包括 `*.java`, `*.go`, `*.xml`, `Makefile` 等 +- LICENSE 文件是否存在 +- NOTICE 文件是否存在 +- CHANGE.md 是否存在 （变更内容格式符合规范） + +以上可以参考其他已发布项目的配置。 + + +``` + +# NOTICE: 这里切分支，分支名称不要和版本号（tag用）类似，不然会有冲突 +$ git checkout -b 1.4 + +$ git tag -a v1.4.0-rc1 -m "v1.4.0 release candidate 1" + +$ git push --tags + +# 打包 +$ git archive --format=tar 1.4 --prefix=dubbo-go-hessian2-v1.4.0/ | gzip > dubbo-go-hessian2-v1.4.0-src.tar.gz + +# 签名 +$ gpg -u wongoo@apache.org --armor --output dubbo-go-hessian2-v1.4.0-src.tar.gz.asc --detach-sign dubbo-go-hessian2-v1.4.0-src.tar.gz + +# 验证签名 +$ gpg --verify dubbo-go-hessian2-v1.4.0-src.tar.gz.asc dubbo-go-hessian2-v1.4.0-src.tar.gz + +# hash +$ shasum -a 512 dubbo-go-hessian2-v1.4.0-src.tar.gz > dubbo-go-hessian2-v1.4.0-src.tar.gz.sha512 + +# 验证 hash +$ shasum --check dubbo-go-hessian2-v1.4.0-src.tar.gz.sha512 + +``` + +> 发布版本: http://www.apache.org/dev/release-publishing.html + +## 3. 上传打包文件到svn仓库 + +``` +$ svn checkout https://dist.apache.org/repos/dist/dev/dubbo + +$ cd dubbo + +# 更新 +$ svn update + +# 添加 签名 和 public key 到KEYS文件并提交到SVN仓库 +# 这里是将公钥KEYS放到根目录, 有的项目放到本次打包文件目录 +$ (gpg --list-sigs wongoo && gpg --armor --export wongoo) >> KEYS + +$ mkdir -p dubbo-go-hessian2/v1.4.0-rc1 + +# 拷贝相关文件到新建目录下 + +$ tree dubbo-go-hessian2 +dubbo-go-hessian2 +└── v1.4.0-rc1 + ├── dubbo-go-hessian2-v1.4.0-src.tar.gz + ├── dubbo-go-hessian2-v1.4.0-src.tar.gz.asc + └── dubbo-go-hessian2-v1.4.0-src.tar.gz.sha512 + +$ svn add dubbo-go-hessian2 +$ svn add dubbo-go-hessian2/* +$ svn status +$ svn commit --username wongoo -m "Release dubbo-go-hessian2 v1.4.0-rc1" +``` + +> 详细参考: svn版本管理 https://www.apache.org/dev/version-control.html + + +## 4. 发投票 [VOTE] 邮件 + +发任何邮件都是有一定格式的，你加入社区邮件列表后，就会收到很多这样的邮件，多看看就知道了，具体邮件范本参考文章后面的邮件范本。 + +发完【VOTE】邮件，私下沟通群里面请大佬PMC投票。 +PMC投票会对你上传打包文件进行相关检查, +详细可以了解孵化中的项目发布完整的检查项参考： https://cwiki.apache.org/confluence/display/INCUBATOR2/IncubatorReleaseChecklist + +收到3个binding邮件且超过72小时后，就可以发 投票结果 [RESULT] [VOTE] 邮件了。 + +> 原则上只有PMC的投票才算binding邮件, 当然也可以由社区决定。 + +这一步骤最常见有以下问题： +- 文件签名有问题 +- 引用项目LICENSE问题 +- 单元测试不通过 + +> 另外需要注意: 一个apache项目可能包含很多子项目，项目的PMC可能只对主项目比较了解， 他们并不清楚如何将子项目跑起来，也不知道如何跑单元测试，最好在邮件中附带一个如何进行单元测试的连接。例如 PMC 最了解 java，但子项目是golang，python，js等，你需要告诉他们如何测试你的项目。 + +可以参考投票规则: https://www.apache.org/foundation/voting.html + +## 5. 发布版本 + +当正式发布投票成功后，先发[Result]邮件，然后就准备 release package。 +将之前在dev下发布的对应rc文件夹下的源码包、签名文件和hash文件拷贝到另一个目录 v1.4.0， +注意文件名字中不要rcxx (可以rename，但不要重新计算签名，hash可以重新计算，结果不会变)。 + +将release包移动到正式版目录。如果你的软件是需要客户从apache下载的，则这一步是必须的。如果不是，比如golang引用github打包地址的则可以忽略。 +``` +svn up +cd dubbo-go-hessian2 +svn move v1.4.0-rc1 v1.4.0 +svn status +svn commit --username wongoo -m "Release dubbo-go-hessian2 v1.4.0" +``` + +移到发版目录后，还需要进行相应的正式版本发布， 这里将具体发布方式整理到单独的章节 `7. 不同语言版本发布`，因为发布流程马上就要结束了 ^v^ + + +## 6. 新版本通告 ANNOUNCE 邮件 + +恭喜你你已经到发版最后一步了，邮件格式参考以下邮件范本！ + + +## 7. 不同语言版本发布 + +### 7.1 golang + +在 github 基于投票分支发布了 release 版本。 + +### 7.2 java + +java项目发版需发布到java maven仓库。 + +TODO + +### 7.3 js + +js项目发版需发布到npm仓库。 + +TODO + +### 7.4 python + +TODO + +## 8. 邮件范本 + +### 8.1. 提出发版投票 + +- TO: dev@dubbo.apache.org +- Title: [VOTE] Release Apache dubbo-go-hessian2 v1.4.0 RC1 + +``` +Hello Dubbo/Dubbogo Community, + + This is a call for vote to release Apache dubbo-go-hessian2 version v1.4.0 RC1. + + The release candidates: https://dist.apache.org/repos/dist/dev/dubbo/dubbo-go-hessian2/v1.4.0-rc1/ + Git tag for the release: https://github.com/apache/dubbo-go-hessian2/tree/1.4 + Hash for the release tag: 4c31e88c35afe84c0321d9f12f036e6d3c8962d0 + Release Notes: https://github.com/apache/dubbo-go-hessian2/blob/1.4/CHANGE.md + The artifacts have been signed with Key :7DB68550D366E4C0, which can be found in the keys file: + https://dist.apache.org/repos/dist/dev/dubbo/KEYS + + The vote will be open for at least 72 hours or until necessary number of votes are reached. + + Please vote accordingly: + [ ] +1 approve + [ ] +0 no opinion + [ ] -1 disapprove with the reason + + Thanks, + The Apache Dubbo-go Team + ``` + + +### 8.2. PMC 投票邮件回复 + + +范例1： +``` ++1 approve <-- 首先表明同不同意 + +I have checked: <-- 其次要说明自己检查了哪些项 + +1.source code can build <-- 能否构建 +2.tests can pass in my local <-- 单元测试能否通过 +3. NOTICE LICENSE file exist <-- 协议文件是否存在 +4.git tag is correct <-- git tag 是否正确 + +there is one minor thing that in change logs file, there is no space +between text And link. I suggest add one to make it looks better. <--- 一些其他改进建议 +``` + +范例2： +``` ++1 + +I checked the following items: + +[v] Are release files in correct location? <-- 发布文件目录是否正确 +[v] Do release files have the word incubating in their name? +[v] Are the digital signature and hashes correct? <-- 签名、hash是否正确 +[v] Do LICENSE and NOTICE files exists? +[v] Is the LICENSE and NOTICE text correct? <-- 协议文本是否正确 +[v] Is the NOTICE year correct? <-- 注意年份是否正确 +[v] Un-included software dependencies are not mentioned in LICENSE or NOTICE? <-- 没有包含协议或注意没有提到的软件依赖 +[v] License information is not mentioned in NOTICE? <-- 协议信息没有在注意中提及 +[x] Is there any 3rd party code contained inside the release? If so: <-- 是否包含第三方代码 + [ ] Does the software have a compatible license? + [ ] Are all software licenses mentioned in LICENSE? + [ ] Is the full text of the licenses (or pointers to it) in LICENSE? + Is any of this code Apache licensed? Do they have NOTICE files? If so: + [ ] Have relevant parts of those NOTICE files been added to this NOTICE file? +[v] Do all source files have ASF headers? <-- 是否所有源码都有ASF头部 +[v] Do the contents of the release match with what's tagged in version control? <-- 发布的文件是否和github中tag标记的版本一致 +[x] Are there any unexpected binary files in the release? <-- 是否包含不应该存在的二进制文件 +[v] Can you compile from source? Are the instruction clear? <-- 能否编译？指令是否明确？ + +On my mac laptop, I could compile successfully but there's one failed unit +test against net.go. I believe this issue [1] can be fixed with [2] in the +next release. <-- 编译问题及建议 + +Is the issue minor? <-- 编译存在的问题是否都是较小的？ +[v] Yes [ ] No [ ] Unsure + +Could it possibly be fixed in the next release? <-- 能否在下一版本修复？ +[v] Yes [ ] No [ ] Unsure + +I vote with: <-- 我的投票 +[v] +1 release the software +[ ] +0 not sure if it should be released +[ ] -1 don’t release the software because... + +Regards, +-Ian. + +1. https://github.com/apache/dubbo-go/issues/207 +2. https://github.com/apache/dubbo-go/pull/209 +``` + +范例3： +``` ++1 + +I checked the following items: + +[√] Do LICENSE and NOTICE files exists? +[√] Is the LICENSE and NOTICE text correct? +[√] Is the NOTICE year correct? +[√] Do all source files have ASF headers? +[√] Do the contents of the release match with what's tagged in version control? +[√] Can you compile from source? +I could compile successfully but there's failed units test. I run the unit +test refer to :https://github.com/apache/dubbo-go#running-unit-tests . +But I think it is not matter, the test can be fixed in next release. + + +I vote with: +[√] +1 release the software +``` + +范例4: +``` +Great improvement over the previous release but there are still issues from the last vote that have not been resolved. e.g. [6][7][8] + +Can someone tell me if these files [1][2][3][4][5] are just missing ASF headers or have a different license? + +If they are just missing headers and [6][7][8] explained then it +1 form me, otherwise it’s probably a -1. + +Can people please carefully check the contents, and write down what you checked, rather than just saying +1. + +I checked: +- signatures and hashes good +- LICENSE is missing the appendix (not a major issue) +- LICENSE may be is missing some information[1][2][3][4][5] +- NOTICE is fine +- No binaries in source release +- Some files are missing ASF headers or other license headers [1][2][3][4][5] - please fix + +Thanks, +Justin + +1. dubbo-go-1.1.0/cluster/loadbalance/round_robin_test.go +2. dubbo-go-1.1.0/common/extension/router_factory.go +3. dubbo-go-1.1.0/config_center/configuration_parser.go +4. dubbo-go-1.1.0/config_center/configuration_parser_test.go +5. dubbo-go-1.1.0/registry/zookeeper/listener_test.go +6. dubbo-go-1.1.0/cluster/loadbalance/least_active.go +7. dubbo-go-1.1.0/protocol/RpcStatus.go +8. dubbo-go-1.1.0/filter/impl/active_filter.go +``` + + +### 8.3. 发 [RESULT] [VOTE] 投票结果通知邮件 + +- TO: dev@dubbo.apache.org +- Title: [RESULT] [VOTE]: Release Apache dubbo-go-hessian2 v1.4.0 RC1 + + +``` +Hello Dubbo/Dubbogo Community, + +The release dubbo-go-hessian2 v1.4.0 RC1 vote finished, We’ve received 3 +1 (binding) votes. + ++1 binding, Stocks Alex ++1 binding, Ian Luo ++1 binding, Jun Liu + +The vote and result thread: +https://lists.apache.org/thread.html/r8070f3b00984888069dd4ddad1bbc424cde51ea68b6ff0520e609e18%40%3Cdev.dubbo.apache.org%3E + + +The vote passed. Thanks all. +I will proceed with the formal release later. + + +Best regards, + +The Apache Dubbogo Team +``` + + +### 8.4. 发 Announce 发版邮件 + +- TO: dev@dubbo.apache.org +- [ANNOUNCE] Apache Dubbo version 2.7.4 Released + +``` +Hello Community, + +The Apache Dubbo team is pleased to announce that the 2.7.4 has been +released. + +Apache Dubbo™ is a high-performance, java based, open source +RPC framework. Dubbo offers three key functionalities, which include +interface based remote call, fault tolerance & load balancing, and +automatic service registration & discovery. + +Both the source release[1] and the maven binary release[2] are available +now, you can also find the detailed release notes in here[3]. + + +If you have any usage questions, or have problems when upgrading or find +any problems about enhancements included in this release, please don’t +hesitate to let us know by sending feedback to this mailing list or filing +an issue on GitHub[4]. + + + +[1] http://dubbo.apache.org/en-us/blog/download.html +[2] http://central.maven.org/maven2/org/apache/dubbo +[3] https://github.com/apache/dubbo/releases +[4] https://github.com/apache/dubbo/issues +``` + +## 9. 参考 + +- dubbo发布流程: http://dubbo.apache.org/zh-cn/docs/developers/committer-guide/release-guide_dev.html +- doris发布流程: https://github.com/apache/incubator-doris/blob/master/docs/documentation/cn/community/release-process.md +- spark发布流程: http://spark0apache0org.icopy.site/release-process.html + + diff --git a/doc/apache/release_note.md b/doc/apache/release_note.md new file mode 100644 index 0000000000..747a3348a1 --- /dev/null +++ b/doc/apache/release_note.md @@ -0,0 +1,11 @@ +### How to release a new version? +--- + +* 1 Check the time range of NOTICE; +* 2 Add the features to the feature list of README.md/README_CN.md/CHANGE.md; +* 3 Check whether every code file has the Apache License 2.0 or not; +* 4 There should not be author info(name & email etc) exist in code file; +* 5 Run all unit tests; +* 6 Run all samples in apache/dubbo-samples/golang; +* 7 Write "What's New" by releaser who should be an apache/dubbo-go committer; +* 8 And then, u can release a new version refer to [Apache 软件发版流程](./apache-release-procedure-20200306.md); \ No newline at end of file diff --git a/go.mod b/go.mod index 8d6647e105..f986ba6adb 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ require ( github.com/Workiva/go-datastructures v1.0.50 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190802083043-4cd0c391755e // indirect - github.com/apache/dubbo-go-hessian2 v1.3.1-0.20200302092433-6ae5479d93a3 + github.com/apache/dubbo-go-hessian2 v1.4.0 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23 // indirect github.com/coreos/bbolt v1.3.3 // indirect github.com/coreos/etcd v3.3.13+incompatible diff --git a/go.sum b/go.sum index ef5649e8c3..3bbae5009b 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190802083043-4cd0c391755e h1:MSuLXx/mveDbpDNhVrcWTMeV4lbYWKcyO4rH+jAxmX0= github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190802083043-4cd0c391755e/go.mod h1:myCDvQSzCW+wB1WAlocEru4wMGJxy+vlxHdhegi1CDQ= github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/apache/dubbo-go-hessian2 v1.3.1-0.20200302092433-6ae5479d93a3 h1:1HM47ILUkLaMxLKUub+WHPncqrJGEQ0KRJzSJueMDpY= -github.com/apache/dubbo-go-hessian2 v1.3.1-0.20200302092433-6ae5479d93a3/go.mod h1:VwEnsOMidkM1usya2uPfGpSLO9XUF//WQcWn3y+jFz8= +github.com/apache/dubbo-go-hessian2 v1.4.0 h1:Cb9FQVTy3G93dnDr7P93U8DeKFYpDTJjQp44JG5TafA= +github.com/apache/dubbo-go-hessian2 v1.4.0/go.mod h1:VwEnsOMidkM1usya2uPfGpSLO9XUF//WQcWn3y+jFz8= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= From 6df03bf99dabef1ef7c61657284582580781a419 Mon Sep 17 00:00:00 2001 From: fangyincheng Date: Sat, 14 Mar 2020 22:37:11 +0800 Subject: [PATCH 14/22] Mod: delete a judgment condition --- config_center/parser/configuration_parser.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/config_center/parser/configuration_parser.go b/config_center/parser/configuration_parser.go index f342dc62e7..e1ddac8e62 100644 --- a/config_center/parser/configuration_parser.go +++ b/config_center/parser/configuration_parser.go @@ -239,12 +239,10 @@ func getParamString(item ConfigItem) (string, error) { retStr = retStr + v } - if len(item.ProviderAddresses) >= 0 { - retStr = retStr + "&" - retStr = retStr + constant.OVERRIDE_PROVIDERS_KEY - retStr = retStr + "=" - retStr = retStr + strings.Join(item.ProviderAddresses, ",") - } + retStr = retStr + "&" + retStr = retStr + constant.OVERRIDE_PROVIDERS_KEY + retStr = retStr + "=" + retStr = retStr + strings.Join(item.ProviderAddresses, ",") return retStr, nil } From 9279a2866b19221166b4a76d13e781e0dd7f0cf7 Mon Sep 17 00:00:00 2001 From: fangyincheng Date: Sat, 14 Mar 2020 22:49:26 +0800 Subject: [PATCH 15/22] Mod: improve code --- config_center/parser/configuration_parser.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/config_center/parser/configuration_parser.go b/config_center/parser/configuration_parser.go index e1ddac8e62..f33b4ba866 100644 --- a/config_center/parser/configuration_parser.go +++ b/config_center/parser/configuration_parser.go @@ -233,16 +233,10 @@ func getParamString(item ConfigItem) (string, error) { "you want to change in the rule.") } for k, v := range params { - retStr = retStr + "&" - retStr = retStr + k - retStr = retStr + "=" - retStr = retStr + v + retStr += "&" + k + "=" + v } - retStr = retStr + "&" - retStr = retStr + constant.OVERRIDE_PROVIDERS_KEY - retStr = retStr + "=" - retStr = retStr + strings.Join(item.ProviderAddresses, ",") + retStr += "&" + constant.OVERRIDE_PROVIDERS_KEY + "=" + strings.Join(item.ProviderAddresses, ",") return retStr, nil } From 9a4db99bb87be3e6e7b4240409b4f2e149d0a6e3 Mon Sep 17 00:00:00 2001 From: Joe Zou Date: Sun, 15 Mar 2020 13:02:03 +0800 Subject: [PATCH 16/22] fix the problem in before_ut.bat --- before_ut.bat | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/before_ut.bat b/before_ut.bat index abe7fc250e..5e1c877af2 100644 --- a/before_ut.bat +++ b/before_ut.bat @@ -14,10 +14,10 @@ :: See the License for the specific language governing permissions and :: limitations under the License. -set zkJarName="zookeeper-3.4.9-fatjar.jar" +set zkJarName=zookeeper-3.4.9-fatjar.jar set remoteJarUrl="https://github.com/dubbogo/resources/raw/master/zookeeper-4unitest/contrib/fatjar/%zkJarName%" -set zkJarPath="remoting/zookeeper/zookeeper-4unittest/contrib/fatjar" -set zkJar="%zkJarPath%/%zkJarName%" +set zkJarPath=remoting/zookeeper/zookeeper-4unittest/contrib/fatjar +set zkJar=%zkJarPath%/%zkJarName% if not exist "%zkJar%" ( md %zkJarPath% From 69de204bf3d7699ec2d7b858b4c8059dde7af4cd Mon Sep 17 00:00:00 2001 From: fangyincheng Date: Sun, 15 Mar 2020 14:59:45 +0800 Subject: [PATCH 17/22] Ref: refact config loader --- ...rest_config_reader.go => confit_reader.go} | 22 +++-- common/yaml/yaml.go | 6 +- common/yaml/yaml_test.go | 9 +- config/config_loader.go | 8 -- config/consumer_config.go | 24 ++++- config/interfaces/config_reader.go | 9 ++ config/provider_config.go | 23 ++++- .../reader_impl/default_config_reader.go | 83 ----------------- .../rest/config_reader/rest_config_reader.go | 25 ------ config/rest_config_loader_test.go | 68 -------------- .../rest/config/reader/rest_config_reader.go | 89 ++++++++----------- .../config/reader/rest_config_reader_test.go | 34 ++++--- .../reader}/testdata/consumer_config.yml | 0 .../reader}/testdata/provider_config.yml | 0 .../rest/config}/rest_config.go | 41 ++++++++- protocol/rest/rest_invoker.go | 6 +- protocol/rest/rest_invoker_test.go | 31 ++++--- protocol/rest/rest_protocol.go | 5 +- protocol/rest/rest_protocol_test.go | 18 ++-- protocol/rest/server/rest_server.go | 6 +- .../server/server_impl/go_restful_server.go | 12 +-- 21 files changed, 201 insertions(+), 318 deletions(-) rename common/extension/{rest_config_reader.go => confit_reader.go} (61%) create mode 100644 config/interfaces/config_reader.go delete mode 100644 config/rest/config_reader/reader_impl/default_config_reader.go delete mode 100644 config/rest/config_reader/rest_config_reader.go delete mode 100644 config/rest_config_loader_test.go rename config/rest_config_loader.go => protocol/rest/config/reader/rest_config_reader.go (60%) rename config/rest/config_reader/reader_impl/default_config_reader_test.go => protocol/rest/config/reader/rest_config_reader_test.go (55%) rename {config/rest/config_reader/reader_impl => protocol/rest/config/reader}/testdata/consumer_config.yml (100%) rename {config/rest/config_reader/reader_impl => protocol/rest/config/reader}/testdata/provider_config.yml (100%) rename {config/rest => protocol/rest/config}/rest_config.go (83%) diff --git a/common/extension/rest_config_reader.go b/common/extension/confit_reader.go similarity index 61% rename from common/extension/rest_config_reader.go rename to common/extension/confit_reader.go index 1bd338cc20..fe7124cf42 100644 --- a/common/extension/rest_config_reader.go +++ b/common/extension/confit_reader.go @@ -18,24 +18,22 @@ package extension import ( - "github.com/apache/dubbo-go/config/rest/config_reader" + "github.com/apache/dubbo-go/config/interfaces" ) var ( - restConfigReaders = make(map[string]func() config_reader.RestConfigReader) + configReaders = make(map[string]func() interfaces.ConfigReader) ) -func SetRestConfigReader(name string, fun func() config_reader.RestConfigReader) { - restConfigReaders[name] = fun +// SetConfigReaders set a creator of config reader. +func SetConfigReaders(name string, v func() interfaces.ConfigReader) { + configReaders[name] = v } -func GetSingletonRestConfigReader(name string) config_reader.RestConfigReader { - if name == "" { - name = "default" +// GetConfigReaders get a config reader by name. +func GetConfigReaders(name string) interfaces.ConfigReader { + if configReaders[name] == nil { + panic("config reader for " + name + " is not existing, make sure you have imported the package.") } - if restConfigReaders[name] == nil { - panic("restConfigReaders for " + name + " is not existing, make sure you have import the package.") - } - return restConfigReaders[name]() - + return configReaders[name]() } diff --git a/common/yaml/yaml.go b/common/yaml/yaml.go index 1d8ac65e90..7c31d71c35 100644 --- a/common/yaml/yaml.go +++ b/common/yaml/yaml.go @@ -41,10 +41,10 @@ func LoadYMLConfig(confProFile string) ([]byte, error) { } // unmarshalYMLConfig Load yml config byte from file , then unmarshal to object -func UnmarshalYMLConfig(confProFile string, out interface{}) error { +func UnmarshalYMLConfig(confProFile string, out interface{}) ([]byte, error) { confFileStream, err := LoadYMLConfig(confProFile) if err != nil { - return perrors.Errorf("ioutil.ReadFile(file:%s) = error:%v", confProFile, perrors.WithStack(err)) + return confFileStream, perrors.Errorf("ioutil.ReadFile(file:%s) = error:%v", confProFile, perrors.WithStack(err)) } - return yaml.Unmarshal(confFileStream, out) + return confFileStream, yaml.Unmarshal(confFileStream, out) } diff --git a/common/yaml/yaml_test.go b/common/yaml/yaml_test.go index 8d0eefddc2..a239ac7b6c 100644 --- a/common/yaml/yaml_test.go +++ b/common/yaml/yaml_test.go @@ -13,7 +13,8 @@ func TestUnmarshalYMLConfig(t *testing.T) { conPath, err := filepath.Abs("./testdata/config.yml") assert.NoError(t, err) c := &Config{} - assert.NoError(t, UnmarshalYMLConfig(conPath, c)) + _, err = UnmarshalYMLConfig(conPath, c) + assert.NoError(t, err) assert.Equal(t, "strTest", c.StrTest) assert.Equal(t, 11, c.IntTest) assert.Equal(t, false, c.BooleanTest) @@ -22,8 +23,10 @@ func TestUnmarshalYMLConfig(t *testing.T) { func TestUnmarshalYMLConfig_Error(t *testing.T) { c := &Config{} - assert.Error(t, UnmarshalYMLConfig("./testdata/config", c)) - assert.Error(t, UnmarshalYMLConfig("", c)) + _, err := UnmarshalYMLConfig("./testdata/config", c) + assert.Error(t, err) + _, err = UnmarshalYMLConfig("", c) + assert.Error(t, err) } type Config struct { diff --git a/config/config_loader.go b/config/config_loader.go index 138106541c..14acc7bd73 100644 --- a/config/config_loader.go +++ b/config/config_loader.go @@ -90,10 +90,6 @@ func Load() { if consumerConfig == nil { logger.Warnf("consumerConfig is nil!") } else { - // init rest consumer config - if err := ConsumerRestConfigInit(consumerConfig.RestConfigType); err != nil { - log.Printf("[initConsumerRestConfig] %#v", err) - } metricConfig = consumerConfig.MetricConfig applicationConfig = consumerConfig.ApplicationConfig @@ -154,10 +150,6 @@ func Load() { if providerConfig == nil { logger.Warnf("providerConfig is nil!") } else { - // init rest provider config - if err := ProviderRestConfigInit(providerConfig.RestConfigType); err != nil { - log.Printf("[initProviderRestConfig] %#v", err) - } // so, you should know that the consumer's config will be override metricConfig = providerConfig.MetricConfig applicationConfig = providerConfig.ApplicationConfig diff --git a/config/consumer_config.go b/config/consumer_config.go index c3fe12d703..21034e0dd5 100644 --- a/config/consumer_config.go +++ b/config/consumer_config.go @@ -18,6 +18,10 @@ package config import ( + "bytes" + "fmt" + "github.com/apache/dubbo-go/common/extension" + "strings" "time" ) @@ -58,7 +62,7 @@ type ConsumerConfig struct { ProtocolConf interface{} `yaml:"protocol_conf" json:"protocol_conf,omitempty" property:"protocol_conf"` FilterConf interface{} `yaml:"filter_conf" json:"filter_conf,omitempty" property:"filter_conf" ` ShutdownConfig *ShutdownConfig `yaml:"shutdown_conf" json:"shutdown_conf,omitempty" property:"shutdown_conf" ` - RestConfigType string `default:"default" yaml:"rest_config_type" json:"rest_config_type,omitempty" property:"rest_config_type"` + ConfigType string `default:"default" yaml:"config_type" json:"config_type,omitempty" property:"config_type"` } // UnmarshalYAML ... @@ -89,7 +93,7 @@ func ConsumerInit(confConFile string) error { return perrors.Errorf("application configure(consumer) file name is nil") } consumerConfig = &ConsumerConfig{} - err := yaml.UnmarshalYMLConfig(confConFile, consumerConfig) + fileStream, err := yaml.UnmarshalYMLConfig(confConFile, consumerConfig) if err != nil { return perrors.Errorf("unmarshalYmlConfig error %v", perrors.WithStack(err)) } @@ -116,6 +120,19 @@ func ConsumerInit(confConFile string) error { } } logger.Debugf("consumer config{%#v}\n", consumerConfig) + + // init other consumer config + conConfigType := consumerConfig.ConfigType + if len(conConfigType) > 0 { + for _, t := range strings.Split(conConfigType, ",") { + if len(t) > 0 { + if err = extension.GetConfigReaders(t).ReadConsumerConfig(bytes.NewBuffer(fileStream)); err != nil { + return perrors.New(fmt.Sprintf("ReadConsumerConfig error: %v for %s", perrors.WithStack(err), t)) + } + } + } + } + return nil } @@ -139,5 +156,6 @@ func configCenterRefreshConsumer() error { return perrors.WithMessagef(err, "time.ParseDuration(Connect_Timeout{%#v})", consumerConfig.Connect_Timeout) } } - return err + + return nil } diff --git a/config/interfaces/config_reader.go b/config/interfaces/config_reader.go new file mode 100644 index 0000000000..23f2225e1b --- /dev/null +++ b/config/interfaces/config_reader.go @@ -0,0 +1,9 @@ +package interfaces + +import "bytes" + +// ConfigReader +type ConfigReader interface { + ReadConsumerConfig(reader *bytes.Buffer) error + ReadProviderConfig(reader *bytes.Buffer) error +} diff --git a/config/provider_config.go b/config/provider_config.go index d8562863ed..a45228f0a4 100644 --- a/config/provider_config.go +++ b/config/provider_config.go @@ -17,6 +17,12 @@ package config +import ( + "bytes" + "fmt" + "strings" +) + import ( "github.com/creasty/defaults" perrors "github.com/pkg/errors" @@ -24,6 +30,7 @@ import ( import ( "github.com/apache/dubbo-go/common/constant" + "github.com/apache/dubbo-go/common/extension" "github.com/apache/dubbo-go/common/logger" "github.com/apache/dubbo-go/common/yaml" ) @@ -46,7 +53,7 @@ type ProviderConfig struct { ProtocolConf interface{} `yaml:"protocol_conf" json:"protocol_conf,omitempty" property:"protocol_conf" ` FilterConf interface{} `yaml:"filter_conf" json:"filter_conf,omitempty" property:"filter_conf" ` ShutdownConfig *ShutdownConfig `yaml:"shutdown_conf" json:"shutdown_conf,omitempty" property:"shutdown_conf" ` - RestConfigType string `default:"default" yaml:"rest_config_type" json:"rest_config_type,omitempty" property:"rest_config_type"` + ConfigType string `default:"default" yaml:"config_type" json:"config_type,omitempty" property:"config_type"` } // UnmarshalYAML ... @@ -77,7 +84,7 @@ func ProviderInit(confProFile string) error { return perrors.Errorf("application configure(provider) file name is nil") } providerConfig = &ProviderConfig{} - err := yaml.UnmarshalYMLConfig(confProFile, providerConfig) + fileStream, err := yaml.UnmarshalYMLConfig(confProFile, providerConfig) if err != nil { return perrors.Errorf("unmarshalYmlConfig error %v", perrors.WithStack(err)) } @@ -92,6 +99,18 @@ func ProviderInit(confProFile string) error { logger.Debugf("provider config{%#v}\n", providerConfig) + // init other provider config + proConfigType := providerConfig.ConfigType + if len(proConfigType) > 0 { + for _, t := range strings.Split(proConfigType, ",") { + if len(t) > 0 { + if err = extension.GetConfigReaders(t).ReadProviderConfig(bytes.NewBuffer(fileStream)); err != nil { + return perrors.New(fmt.Sprintf("ReadProviderConfig error: %v for %s", perrors.WithStack(err), t)) + } + } + } + } + return nil } diff --git a/config/rest/config_reader/reader_impl/default_config_reader.go b/config/rest/config_reader/reader_impl/default_config_reader.go deleted file mode 100644 index 40e1ecf9a0..0000000000 --- a/config/rest/config_reader/reader_impl/default_config_reader.go +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package reader_impl - -import ( - "github.com/apache/dubbo-go/config/rest" - "github.com/apache/dubbo-go/config/rest/config_reader" - "os" -) - -import ( - perrors "github.com/pkg/errors" -) - -import ( - "github.com/apache/dubbo-go/common/constant" - "github.com/apache/dubbo-go/common/extension" - "github.com/apache/dubbo-go/common/yaml" -) - -var ( - defaultConfigReader *DefaultConfigReader -) - -func init() { - extension.SetRestConfigReader(constant.DEFAULT_KEY, GetDefaultConfigReader) -} - -type DefaultConfigReader struct { -} - -func NewDefaultConfigReader() *DefaultConfigReader { - return &DefaultConfigReader{} -} - -func (dcr *DefaultConfigReader) ReadConsumerConfig() (*rest.RestConsumerConfig, error) { - confConFile := os.Getenv(constant.CONF_CONSUMER_FILE_PATH) - if len(confConFile) == 0 { - return nil, nil - } - restConsumerConfig := &rest.RestConsumerConfig{} - err := yaml.UnmarshalYMLConfig(confConFile, restConsumerConfig) - if err != nil { - return nil, perrors.Errorf("[Rest Config] unmarshal Consumer RestYmlConfig error %v", perrors.WithStack(err)) - } - return restConsumerConfig, nil -} - -func (dcr *DefaultConfigReader) ReadProviderConfig() (*rest.RestProviderConfig, error) { - confProFile := os.Getenv(constant.CONF_PROVIDER_FILE_PATH) - if len(confProFile) == 0 { - return nil, nil - } - restProviderConfig := &rest.RestProviderConfig{} - err := yaml.UnmarshalYMLConfig(confProFile, restProviderConfig) - if err != nil { - return nil, perrors.Errorf("[Rest Config] unmarshal Provider RestYmlConfig error %v", perrors.WithStack(err)) - - } - return restProviderConfig, nil -} - -func GetDefaultConfigReader() config_reader.RestConfigReader { - if defaultConfigReader == nil { - defaultConfigReader = NewDefaultConfigReader() - } - return defaultConfigReader -} diff --git a/config/rest/config_reader/rest_config_reader.go b/config/rest/config_reader/rest_config_reader.go deleted file mode 100644 index 366cde6a0d..0000000000 --- a/config/rest/config_reader/rest_config_reader.go +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package config_reader - -import "github.com/apache/dubbo-go/config/rest" - -type RestConfigReader interface { - ReadConsumerConfig() (*rest.RestConsumerConfig, error) - ReadProviderConfig() (*rest.RestProviderConfig, error) -} diff --git a/config/rest_config_loader_test.go b/config/rest_config_loader_test.go deleted file mode 100644 index 7b2491a501..0000000000 --- a/config/rest_config_loader_test.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package config - -import ( - "os" - "testing" -) - -import ( - "github.com/stretchr/testify/assert" -) - -import ( - "github.com/apache/dubbo-go/common/constant" -) - -func TestGetRestConsumerServiceConfig(t *testing.T) { - err := os.Setenv(constant.CONF_CONSUMER_FILE_PATH, "./rest/config_reader/reader_impl/testdata/consumer_config.yml") - assert.NoError(t, err) - err = ConsumerRestConfigInit("default") - assert.NoError(t, err) - serviceConfig := GetRestConsumerServiceConfig("UserProvider") - assert.NotEmpty(t, serviceConfig) - assert.NotEmpty(t, serviceConfig.RestMethodConfigsMap) - assert.NotEmpty(t, serviceConfig.RestMethodConfigsMap["GetUser"]) - assert.Equal(t, serviceConfig.RestMethodConfigsMap["GetUser"].QueryParamsMap[1], "userid") - assert.Equal(t, serviceConfig.RestMethodConfigsMap["GetUser"].HeadersMap[3], "age") - assert.Equal(t, serviceConfig.RestMethodConfigsMap["GetUser"].PathParamsMap[4], "time") - assert.Equal(t, serviceConfig.RestMethodConfigsMap["GetUser"].Body, 0) - assert.Equal(t, serviceConfig.RestMethodConfigsMap["GetUser"].Produces, "application/xml") - assert.Equal(t, serviceConfig.RestMethodConfigsMap["GetUser"].Consumes, "application/xml") - assert.Equal(t, serviceConfig.Client, "resty1") -} - -func TestGetRestProviderServiceConfig(t *testing.T) { - err := os.Setenv(constant.CONF_PROVIDER_FILE_PATH, "./rest/config_reader/reader_impl/testdata/provider_config.yml") - assert.NoError(t, err) - err = ProviderRestConfigInit("default") - assert.NoError(t, err) - serviceConfig := GetRestProviderServiceConfig("UserProvider") - assert.NotEmpty(t, serviceConfig) - assert.NotEmpty(t, serviceConfig.RestMethodConfigsMap) - assert.NotEmpty(t, serviceConfig.RestMethodConfigsMap["GetUser"]) - assert.Equal(t, serviceConfig.RestMethodConfigsMap["GetUser"].QueryParamsMap[1], "userid") - assert.Equal(t, serviceConfig.RestMethodConfigsMap["GetUser"].HeadersMap[3], "age") - assert.Equal(t, serviceConfig.RestMethodConfigsMap["GetUser"].PathParamsMap[4], "time") - assert.Equal(t, serviceConfig.RestMethodConfigsMap["GetUser"].Body, 0) - assert.Equal(t, serviceConfig.RestMethodConfigsMap["GetUser"].Produces, "application/xml") - assert.Equal(t, serviceConfig.RestMethodConfigsMap["GetUser"].Consumes, "application/xml") - assert.Equal(t, serviceConfig.Server, "go-restful1") - -} diff --git a/config/rest_config_loader.go b/protocol/rest/config/reader/rest_config_reader.go similarity index 60% rename from config/rest_config_loader.go rename to protocol/rest/config/reader/rest_config_reader.go index 6596f7b274..ff19e97e2e 100644 --- a/config/rest_config_loader.go +++ b/protocol/rest/config/reader/rest_config_reader.go @@ -15,73 +15,78 @@ * limitations under the License. */ -package config +package reader import ( + "bytes" + "github.com/apache/dubbo-go/protocol/rest/config" "strconv" "strings" ) import ( perrors "github.com/pkg/errors" + "gopkg.in/yaml.v2" ) import ( "github.com/apache/dubbo-go/common/constant" "github.com/apache/dubbo-go/common/extension" "github.com/apache/dubbo-go/common/logger" - "github.com/apache/dubbo-go/config/rest" - _ "github.com/apache/dubbo-go/config/rest/config_reader/reader_impl" + "github.com/apache/dubbo-go/config/interfaces" ) -var ( - restConsumerServiceConfigMap map[string]*rest.RestServiceConfig - restProviderServiceConfigMap map[string]*rest.RestServiceConfig -) +const REST = "rest" -// initConsumerRestConfig ... -func ConsumerRestConfigInit(configType string) error { - consumerConfigReader := extension.GetSingletonRestConfigReader(configType) - var restConsumerConfig *rest.RestConsumerConfig - var err error - if restConsumerConfig, err = consumerConfigReader.ReadConsumerConfig(); err != nil { - return err - } - if restConsumerConfig == nil || len(restConsumerConfig.RestServiceConfigsMap) == 0 { - return perrors.New("Consumer don't has RestServiceConfigsMap ") +func init() { + extension.SetConfigReaders(REST, NewRestConfigReader) +} + +type RestConfigReader struct { +} + +func NewRestConfigReader() interfaces.ConfigReader { + return &RestConfigReader{} +} + +// ReadConsumerConfig read consumer config for rest protocol +func (cr *RestConfigReader) ReadConsumerConfig(reader *bytes.Buffer) error { + restConsumerConfig := &config.RestConsumerConfig{} + err := yaml.Unmarshal(reader.Bytes(), restConsumerConfig) + if err != nil { + return perrors.Errorf("[Rest Config] unmarshal Consumer error %#v", perrors.WithStack(err)) } - restConsumerServiceConfigMap = make(map[string]*rest.RestServiceConfig, len(restConsumerConfig.RestServiceConfigsMap)) + + restConsumerServiceConfigMap := make(map[string]*config.RestServiceConfig, len(restConsumerConfig.RestServiceConfigsMap)) for key, rc := range restConsumerConfig.RestServiceConfigsMap { rc.Client = getNotEmptyStr(rc.Client, restConsumerConfig.Client, constant.DEFAULT_REST_CLIENT) rc.RestMethodConfigsMap = initMethodConfigMap(rc, restConsumerConfig.Consumes, restConsumerConfig.Produces) restConsumerServiceConfigMap[strings.TrimPrefix(key, "/")] = rc } + config.SetRestConsumerServiceConfigMap(restConsumerServiceConfigMap) return nil } -// initProviderRestConfig ... -func ProviderRestConfigInit(configType string) error { - providerConfigReader := extension.GetSingletonRestConfigReader(configType) - var restProviderConfig *rest.RestProviderConfig - var err error - if restProviderConfig, err = providerConfigReader.ReadProviderConfig(); err != nil { - return err +// ReadProviderConfig read provider config for rest protocol +func (cr *RestConfigReader) ReadProviderConfig(reader *bytes.Buffer) error { + restProviderConfig := &config.RestProviderConfig{} + err := yaml.Unmarshal(reader.Bytes(), restProviderConfig) + if err != nil { + return perrors.Errorf("[Rest Config] unmarshal Provider error %#v", perrors.WithStack(err)) } - if restProviderConfig == nil || len(restProviderConfig.RestServiceConfigsMap) == 0 { - return perrors.New("Provider don't has RestServiceConfigsMap") - } - restProviderServiceConfigMap = make(map[string]*rest.RestServiceConfig, len(restProviderConfig.RestServiceConfigsMap)) + restProviderServiceConfigMap := make(map[string]*config.RestServiceConfig, len(restProviderConfig.RestServiceConfigsMap)) for key, rc := range restProviderConfig.RestServiceConfigsMap { rc.Server = getNotEmptyStr(rc.Server, restProviderConfig.Server, constant.DEFAULT_REST_SERVER) rc.RestMethodConfigsMap = initMethodConfigMap(rc, restProviderConfig.Consumes, restProviderConfig.Produces) restProviderServiceConfigMap[strings.TrimPrefix(key, "/")] = rc } + config.SetRestProviderServiceConfigMap(restProviderServiceConfigMap) return nil } // initProviderRestConfig ... -func initMethodConfigMap(rc *rest.RestServiceConfig, consumes string, produces string) map[string]*rest.RestMethodConfig { - mcm := make(map[string]*rest.RestMethodConfig, len(rc.RestMethodConfigs)) +func initMethodConfigMap(rc *config.RestServiceConfig, consumes string, produces string) map[string]*config.RestMethodConfig { + mcm := make(map[string]*config.RestMethodConfig, len(rc.RestMethodConfigs)) for _, mc := range rc.RestMethodConfigs { mc.InterfaceName = rc.InterfaceName mc.Path = rc.Path + mc.Path @@ -107,7 +112,7 @@ func getNotEmptyStr(args ...string) string { } // transformMethodConfig -func transformMethodConfig(methodConfig *rest.RestMethodConfig) *rest.RestMethodConfig { +func transformMethodConfig(methodConfig *config.RestMethodConfig) *config.RestMethodConfig { if len(methodConfig.PathParamsMap) == 0 && len(methodConfig.PathParams) > 0 { paramsMap, err := parseParamsString2Map(methodConfig.PathParams) if err != nil { @@ -150,23 +155,3 @@ func parseParamsString2Map(params string) (map[int]string, error) { } return m, nil } - -// GetRestConsumerServiceConfig ... -func GetRestConsumerServiceConfig(path string) *rest.RestServiceConfig { - return restConsumerServiceConfigMap[path] -} - -// GetRestProviderServiceConfig ... -func GetRestProviderServiceConfig(path string) *rest.RestServiceConfig { - return restProviderServiceConfigMap[path] -} - -// SetRestConsumerServiceConfigMap ... -func SetRestConsumerServiceConfigMap(configMap map[string]*rest.RestServiceConfig) { - restConsumerServiceConfigMap = configMap -} - -// SetRestProviderServiceConfigMap ... -func SetRestProviderServiceConfigMap(configMap map[string]*rest.RestServiceConfig) { - restProviderServiceConfigMap = configMap -} diff --git a/config/rest/config_reader/reader_impl/default_config_reader_test.go b/protocol/rest/config/reader/rest_config_reader_test.go similarity index 55% rename from config/rest/config_reader/reader_impl/default_config_reader_test.go rename to protocol/rest/config/reader/rest_config_reader_test.go index eca7436a10..a4e09d2d9c 100644 --- a/config/rest/config_reader/reader_impl/default_config_reader_test.go +++ b/protocol/rest/config/reader/rest_config_reader_test.go @@ -15,10 +15,12 @@ * limitations under the License. */ -package reader_impl +package reader import ( - "os" + "bytes" + "github.com/apache/dubbo-go/common/yaml" + "github.com/apache/dubbo-go/protocol/rest/config" "testing" ) @@ -26,24 +28,20 @@ import ( "github.com/stretchr/testify/assert" ) -import ( - "github.com/apache/dubbo-go/common/constant" -) - -func TestDefaultConfigReader_ReadConsumerConfig(t *testing.T) { - err := os.Setenv(constant.CONF_CONSUMER_FILE_PATH, "./testdata/consumer_config.yml") +func TestRestConfigReader_ReadConsumerConfig(t *testing.T) { + bs, err := yaml.LoadYMLConfig("./testdata/consumer_config.yml") assert.NoError(t, err) - reader := GetDefaultConfigReader() - config, err := reader.ReadConsumerConfig() - assert.Nil(t, err) - assert.NotEmpty(t, config) + configReader := NewRestConfigReader() + err = configReader.ReadConsumerConfig(bytes.NewBuffer(bs)) + assert.NoError(t, err) + assert.NotEmpty(t, config.GetRestConsumerServiceConfigMap()) } -func TestDefaultConfigReader_ReadProviderConfig(t *testing.T) { - err := os.Setenv(constant.CONF_PROVIDER_FILE_PATH, "./testdata/provider_config.yml") +func TestRestConfigReader_ReadProviderConfig(t *testing.T) { + bs, err := yaml.LoadYMLConfig("./testdata/provider_config.yml") + assert.NoError(t, err) + configReader := NewRestConfigReader() + err = configReader.ReadProviderConfig(bytes.NewBuffer(bs)) assert.NoError(t, err) - reader := GetDefaultConfigReader() - config, err := reader.ReadProviderConfig() - assert.Nil(t, err) - assert.NotEmpty(t, config) + assert.NotEmpty(t, config.GetRestProviderServiceConfigMap()) } diff --git a/config/rest/config_reader/reader_impl/testdata/consumer_config.yml b/protocol/rest/config/reader/testdata/consumer_config.yml similarity index 100% rename from config/rest/config_reader/reader_impl/testdata/consumer_config.yml rename to protocol/rest/config/reader/testdata/consumer_config.yml diff --git a/config/rest/config_reader/reader_impl/testdata/provider_config.yml b/protocol/rest/config/reader/testdata/provider_config.yml similarity index 100% rename from config/rest/config_reader/reader_impl/testdata/provider_config.yml rename to protocol/rest/config/reader/testdata/provider_config.yml diff --git a/config/rest/rest_config.go b/protocol/rest/config/rest_config.go similarity index 83% rename from config/rest/rest_config.go rename to protocol/rest/config/rest_config.go index 7262776ec3..168ec8ce52 100644 --- a/config/rest/rest_config.go +++ b/protocol/rest/config/rest_config.go @@ -15,9 +15,16 @@ * limitations under the License. */ -package rest +package config -import "github.com/creasty/defaults" +import ( + "github.com/creasty/defaults" +) + +var ( + restConsumerServiceConfigMap map[string]*RestServiceConfig + restProviderServiceConfigMap map[string]*RestServiceConfig +) // RestConsumerConfig ... type RestConsumerConfig struct { @@ -114,3 +121,33 @@ func (c *RestMethodConfig) UnmarshalYAML(unmarshal func(interface{}) error) erro } return nil } + +// GetRestConsumerServiceConfig ... +func GetRestConsumerServiceConfig(path string) *RestServiceConfig { + return restConsumerServiceConfigMap[path] +} + +// GetRestProviderServiceConfig ... +func GetRestProviderServiceConfig(path string) *RestServiceConfig { + return restProviderServiceConfigMap[path] +} + +// SetRestConsumerServiceConfigMap ... +func SetRestConsumerServiceConfigMap(configMap map[string]*RestServiceConfig) { + restConsumerServiceConfigMap = configMap +} + +// SetRestProviderServiceConfigMap ... +func SetRestProviderServiceConfigMap(configMap map[string]*RestServiceConfig) { + restProviderServiceConfigMap = configMap +} + +// GetRestConsumerServiceConfigMap ... +func GetRestConsumerServiceConfigMap() map[string]*RestServiceConfig { + return restConsumerServiceConfigMap +} + +// GetRestProviderServiceConfigMap ... +func GetRestProviderServiceConfigMap() map[string]*RestServiceConfig { + return restProviderServiceConfigMap +} diff --git a/protocol/rest/rest_invoker.go b/protocol/rest/rest_invoker.go index 02efba59d9..0c82035ac5 100644 --- a/protocol/rest/rest_invoker.go +++ b/protocol/rest/rest_invoker.go @@ -28,19 +28,19 @@ import ( import ( "github.com/apache/dubbo-go/common" - "github.com/apache/dubbo-go/config/rest" "github.com/apache/dubbo-go/protocol" invocation_impl "github.com/apache/dubbo-go/protocol/invocation" "github.com/apache/dubbo-go/protocol/rest/client" + "github.com/apache/dubbo-go/protocol/rest/config" ) type RestInvoker struct { protocol.BaseInvoker client client.RestClient - restMethodConfigMap map[string]*rest.RestMethodConfig + restMethodConfigMap map[string]*config.RestMethodConfig } -func NewRestInvoker(url common.URL, client *client.RestClient, restMethodConfig map[string]*rest.RestMethodConfig) *RestInvoker { +func NewRestInvoker(url common.URL, client *client.RestClient, restMethodConfig map[string]*config.RestMethodConfig) *RestInvoker { return &RestInvoker{ BaseInvoker: *protocol.NewBaseInvoker(url), client: *client, diff --git a/protocol/rest/rest_invoker_test.go b/protocol/rest/rest_invoker_test.go index b6eaa17334..bea9aa81d9 100644 --- a/protocol/rest/rest_invoker_test.go +++ b/protocol/rest/rest_invoker_test.go @@ -19,8 +19,6 @@ package rest import ( "context" - "github.com/apache/dubbo-go/protocol/rest/client" - "github.com/apache/dubbo-go/protocol/rest/client/client_impl" "testing" "time" ) @@ -33,9 +31,10 @@ import ( "github.com/apache/dubbo-go/common" "github.com/apache/dubbo-go/common/extension" "github.com/apache/dubbo-go/config" - "github.com/apache/dubbo-go/config/rest" - _ "github.com/apache/dubbo-go/config/rest/config_reader/reader_impl" "github.com/apache/dubbo-go/protocol/invocation" + "github.com/apache/dubbo-go/protocol/rest/client" + "github.com/apache/dubbo-go/protocol/rest/client/client_impl" + rest_config "github.com/apache/dubbo-go/protocol/rest/config" ) func TestRestInvoker_Invoke(t *testing.T) { @@ -52,8 +51,8 @@ func TestRestInvoker_Invoke(t *testing.T) { assert.NoError(t, err) con := config.ProviderConfig{} config.SetProviderConfig(con) - configMap := make(map[string]*rest.RestServiceConfig) - methodConfigMap := make(map[string]*rest.RestMethodConfig) + configMap := make(map[string]*rest_config.RestServiceConfig) + methodConfigMap := make(map[string]*rest_config.RestMethodConfig) queryParamsMap := make(map[int]string) queryParamsMap[1] = "age" queryParamsMap[2] = "name" @@ -61,7 +60,7 @@ func TestRestInvoker_Invoke(t *testing.T) { pathParamsMap[0] = "userid" headersMap := make(map[int]string) headersMap[3] = "Content-Type" - methodConfigMap["GetUserOne"] = &rest.RestMethodConfig{ + methodConfigMap["GetUserOne"] = &rest_config.RestMethodConfig{ InterfaceName: "", MethodName: "GetUserOne", Path: "/GetUserOne", @@ -74,7 +73,7 @@ func TestRestInvoker_Invoke(t *testing.T) { QueryParamsMap: nil, Body: 0, } - methodConfigMap["GetUserTwo"] = &rest.RestMethodConfig{ + methodConfigMap["GetUserTwo"] = &rest_config.RestMethodConfig{ InterfaceName: "", MethodName: "GetUserTwo", Path: "/GetUserTwo", @@ -87,7 +86,7 @@ func TestRestInvoker_Invoke(t *testing.T) { QueryParamsMap: nil, Body: 0, } - methodConfigMap["GetUserThree"] = &rest.RestMethodConfig{ + methodConfigMap["GetUserThree"] = &rest_config.RestMethodConfig{ InterfaceName: "", MethodName: "GetUserThree", Path: "/GetUserThree", @@ -100,7 +99,7 @@ func TestRestInvoker_Invoke(t *testing.T) { QueryParamsMap: nil, Body: 0, } - methodConfigMap["GetUserFour"] = &rest.RestMethodConfig{ + methodConfigMap["GetUserFour"] = &rest_config.RestMethodConfig{ InterfaceName: "", MethodName: "GetUserFour", Path: "/GetUserFour", @@ -113,7 +112,7 @@ func TestRestInvoker_Invoke(t *testing.T) { QueryParamsMap: nil, Body: 0, } - methodConfigMap["GetUserFive"] = &rest.RestMethodConfig{ + methodConfigMap["GetUserFive"] = &rest_config.RestMethodConfig{ InterfaceName: "", MethodName: "GetUserFive", Path: "/GetUserFive", @@ -121,7 +120,7 @@ func TestRestInvoker_Invoke(t *testing.T) { Consumes: "*/*", MethodType: "GET", } - methodConfigMap["GetUser"] = &rest.RestMethodConfig{ + methodConfigMap["GetUser"] = &rest_config.RestMethodConfig{ InterfaceName: "", MethodName: "GetUser", Path: "/GetUser/{userid}", @@ -136,16 +135,16 @@ func TestRestInvoker_Invoke(t *testing.T) { HeadersMap: headersMap, } - configMap["com.ikurento.user.UserProvider"] = &rest.RestServiceConfig{ + configMap["com.ikurento.user.UserProvider"] = &rest_config.RestServiceConfig{ Server: "go-restful", RestMethodConfigsMap: methodConfigMap, } - config.SetRestProviderServiceConfigMap(configMap) + rest_config.SetRestProviderServiceConfigMap(configMap) proxyFactory := extension.GetProxyFactory("default") proto.Export(proxyFactory.GetInvoker(url)) time.Sleep(5 * time.Second) - configMap = make(map[string]*rest.RestServiceConfig) - configMap["com.ikurento.user.UserProvider"] = &rest.RestServiceConfig{ + configMap = make(map[string]*rest_config.RestServiceConfig) + configMap["com.ikurento.user.UserProvider"] = &rest_config.RestServiceConfig{ RestMethodConfigsMap: methodConfigMap, } restClient := client_impl.GetRestyClient(&client.RestOptions{ConnectTimeout: 3 * time.Second, RequestTimeout: 3 * time.Second}) diff --git a/protocol/rest/rest_protocol.go b/protocol/rest/rest_protocol.go index d068d9b6c4..8801697534 100644 --- a/protocol/rest/rest_protocol.go +++ b/protocol/rest/rest_protocol.go @@ -32,6 +32,7 @@ import ( "github.com/apache/dubbo-go/protocol" "github.com/apache/dubbo-go/protocol/rest/client" _ "github.com/apache/dubbo-go/protocol/rest/client/client_impl" + rest_config "github.com/apache/dubbo-go/protocol/rest/config" "github.com/apache/dubbo-go/protocol/rest/server" _ "github.com/apache/dubbo-go/protocol/rest/server/server_impl" ) @@ -66,7 +67,7 @@ func (rp *RestProtocol) Export(invoker protocol.Invoker) protocol.Exporter { url := invoker.GetUrl() serviceKey := url.ServiceKey() exporter := NewRestExporter(serviceKey, invoker, rp.ExporterMap()) - restServiceConfig := config.GetRestProviderServiceConfig(strings.TrimPrefix(url.Path, "/")) + restServiceConfig := rest_config.GetRestProviderServiceConfig(strings.TrimPrefix(url.Path, "/")) if restServiceConfig == nil { logger.Errorf("%s service doesn't has provider config", url.Path) return nil @@ -85,7 +86,7 @@ func (rp *RestProtocol) Refer(url common.URL) protocol.Invoker { if t, err := time.ParseDuration(requestTimeoutStr); err == nil { requestTimeout = t } - restServiceConfig := config.GetRestConsumerServiceConfig(strings.TrimPrefix(url.Path, "/")) + restServiceConfig := rest_config.GetRestConsumerServiceConfig(strings.TrimPrefix(url.Path, "/")) if restServiceConfig == nil { logger.Errorf("%s service doesn't has consumer config", url.Path) return nil diff --git a/protocol/rest/rest_protocol_test.go b/protocol/rest/rest_protocol_test.go index 30d41b352b..8af73a1839 100644 --- a/protocol/rest/rest_protocol_test.go +++ b/protocol/rest/rest_protocol_test.go @@ -35,7 +35,7 @@ import ( "github.com/apache/dubbo-go/common/extension" _ "github.com/apache/dubbo-go/common/proxy/proxy_factory" "github.com/apache/dubbo-go/config" - "github.com/apache/dubbo-go/config/rest" + rest_config "github.com/apache/dubbo-go/protocol/rest/config" ) func TestRestProtocol_Refer(t *testing.T) { @@ -52,11 +52,11 @@ func TestRestProtocol_Refer(t *testing.T) { RequestTimeout: 5 * time.Second, } config.SetConsumerConfig(con) - configMap := make(map[string]*rest.RestServiceConfig) - configMap["com.ikurento.user.UserProvider"] = &rest.RestServiceConfig{ + configMap := make(map[string]*rest_config.RestServiceConfig) + configMap["com.ikurento.user.UserProvider"] = &rest_config.RestServiceConfig{ Client: "resty", } - config.SetRestConsumerServiceConfigMap(configMap) + rest_config.SetRestConsumerServiceConfigMap(configMap) invoker := proto.Refer(url) // make sure url @@ -84,14 +84,14 @@ func TestRestProtocol_Export(t *testing.T) { assert.NoError(t, err) con := config.ProviderConfig{} config.SetProviderConfig(con) - configMap := make(map[string]*rest.RestServiceConfig) - methodConfigMap := make(map[string]*rest.RestMethodConfig) + configMap := make(map[string]*rest_config.RestServiceConfig) + methodConfigMap := make(map[string]*rest_config.RestMethodConfig) queryParamsMap := make(map[int]string) queryParamsMap[1] = "age" queryParamsMap[2] = "name" pathParamsMap := make(map[int]string) pathParamsMap[0] = "userid" - methodConfigMap["GetUser"] = &rest.RestMethodConfig{ + methodConfigMap["GetUser"] = &rest_config.RestMethodConfig{ InterfaceName: "", MethodName: "GetUser", Path: "/GetUser/{userid}", @@ -104,11 +104,11 @@ func TestRestProtocol_Export(t *testing.T) { QueryParamsMap: queryParamsMap, Body: -1, } - configMap["com.ikurento.user.UserProvider"] = &rest.RestServiceConfig{ + configMap["com.ikurento.user.UserProvider"] = &rest_config.RestServiceConfig{ Server: "go-restful", RestMethodConfigsMap: methodConfigMap, } - config.SetRestProviderServiceConfigMap(configMap) + rest_config.SetRestProviderServiceConfigMap(configMap) proxyFactory := extension.GetProxyFactory("default") exporter := proto.Export(proxyFactory.GetInvoker(url)) // make sure url diff --git a/protocol/rest/server/rest_server.go b/protocol/rest/server/rest_server.go index e938c4d1e0..c10c98a7b6 100644 --- a/protocol/rest/server/rest_server.go +++ b/protocol/rest/server/rest_server.go @@ -19,13 +19,13 @@ package server import ( "github.com/apache/dubbo-go/common" - "github.com/apache/dubbo-go/config/rest" "github.com/apache/dubbo-go/protocol" + "github.com/apache/dubbo-go/protocol/rest/config" ) type RestServer interface { Start(url common.URL) - Deploy(invoker protocol.Invoker, restMethodConfig map[string]*rest.RestMethodConfig) - UnDeploy(restMethodConfig map[string]*rest.RestMethodConfig) + Deploy(invoker protocol.Invoker, restMethodConfig map[string]*config.RestMethodConfig) + UnDeploy(restMethodConfig map[string]*config.RestMethodConfig) Destroy() } diff --git a/protocol/rest/server/server_impl/go_restful_server.go b/protocol/rest/server/server_impl/go_restful_server.go index a630a34edb..51f64e1293 100644 --- a/protocol/rest/server/server_impl/go_restful_server.go +++ b/protocol/rest/server/server_impl/go_restful_server.go @@ -20,6 +20,7 @@ package server_impl import ( "context" "fmt" + "github.com/apache/dubbo-go/protocol/rest/config" "github.com/apache/dubbo-go/protocol/rest/server" "net" "net/http" @@ -30,7 +31,6 @@ import ( ) import ( - "github.com/apache/dubbo-go/config/rest" "github.com/emicklei/go-restful/v3" perrors "github.com/pkg/errors" ) @@ -75,7 +75,7 @@ func (grs *GoRestfulServer) Start(url common.URL) { }() } -func (grs *GoRestfulServer) Deploy(invoker protocol.Invoker, restMethodConfig map[string]*rest.RestMethodConfig) { +func (grs *GoRestfulServer) Deploy(invoker protocol.Invoker, restMethodConfig map[string]*config.RestMethodConfig) { svc := common.ServiceMap.GetService(invoker.GetUrl().Protocol, strings.TrimPrefix(invoker.GetUrl().Path, "/")) for methodName, config := range restMethodConfig { // get method @@ -93,7 +93,7 @@ func (grs *GoRestfulServer) Deploy(invoker protocol.Invoker, restMethodConfig ma } func getFunc(methodName string, invoker protocol.Invoker, argsTypes []reflect.Type, - replyType reflect.Type, config *rest.RestMethodConfig) func(req *restful.Request, resp *restful.Response) { + replyType reflect.Type, config *config.RestMethodConfig) func(req *restful.Request, resp *restful.Response) { return func(req *restful.Request, resp *restful.Response) { var ( err error @@ -119,7 +119,7 @@ func getFunc(methodName string, invoker protocol.Invoker, argsTypes []reflect.Ty } } } -func (grs *GoRestfulServer) UnDeploy(restMethodConfig map[string]*rest.RestMethodConfig) { +func (grs *GoRestfulServer) UnDeploy(restMethodConfig map[string]*config.RestMethodConfig) { for _, config := range restMethodConfig { ws := new(restful.WebService) ws.Path(config.Path) @@ -139,7 +139,7 @@ func (grs *GoRestfulServer) Destroy() { logger.Infof("[Go Restful] Server exiting") } -func getArgsInterfaceFromRequest(req *restful.Request, config *rest.RestMethodConfig) []interface{} { +func getArgsInterfaceFromRequest(req *restful.Request, config *config.RestMethodConfig) []interface{} { argsMap := make(map[int]interface{}, 8) maxKey := 0 for k, v := range config.PathParamsMap { @@ -186,7 +186,7 @@ func getArgsInterfaceFromRequest(req *restful.Request, config *rest.RestMethodCo return args } -func getArgsFromRequest(req *restful.Request, argsTypes []reflect.Type, config *rest.RestMethodConfig) []interface{} { +func getArgsFromRequest(req *restful.Request, argsTypes []reflect.Type, config *config.RestMethodConfig) []interface{} { argsLength := len(argsTypes) args := make([]interface{}, argsLength) for i, t := range argsTypes { From 66d8a1ad2774e7dbd95223ef1fe39b89f8b6fffc Mon Sep 17 00:00:00 2001 From: fangyincheng Date: Sun, 15 Mar 2020 16:17:16 +0800 Subject: [PATCH 18/22] Del: delete config/rest --- .../reader_impl/default_config_reader.go | 83 ------------------- 1 file changed, 83 deletions(-) delete mode 100644 config/rest/config_reader/reader_impl/default_config_reader.go diff --git a/config/rest/config_reader/reader_impl/default_config_reader.go b/config/rest/config_reader/reader_impl/default_config_reader.go deleted file mode 100644 index 89cf247483..0000000000 --- a/config/rest/config_reader/reader_impl/default_config_reader.go +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package reader_impl - -import ( - "os" -) - -import ( - perrors "github.com/pkg/errors" -) - -import ( - "github.com/apache/dubbo-go/common/constant" - "github.com/apache/dubbo-go/common/extension" - "github.com/apache/dubbo-go/common/yaml" - "github.com/apache/dubbo-go/config/rest" - "github.com/apache/dubbo-go/config/rest/config_reader" -) - -var ( - defaultConfigReader *DefaultConfigReader -) - -func init() { - extension.SetRestConfigReader(constant.DEFAULT_KEY, GetDefaultConfigReader) -} - -type DefaultConfigReader struct { -} - -func NewDefaultConfigReader() *DefaultConfigReader { - return &DefaultConfigReader{} -} - -func (dcr *DefaultConfigReader) ReadConsumerConfig() (*rest.RestConsumerConfig, error) { - confConFile := os.Getenv(constant.CONF_CONSUMER_FILE_PATH) - if len(confConFile) == 0 { - return nil, nil - } - restConsumerConfig := &rest.RestConsumerConfig{} - err := yaml.UnmarshalYMLConfig(confConFile, restConsumerConfig) - if err != nil { - return nil, perrors.Errorf("[Rest Config] unmarshal Consumer RestYmlConfig error %v", perrors.WithStack(err)) - } - return restConsumerConfig, nil -} - -func (dcr *DefaultConfigReader) ReadProviderConfig() (*rest.RestProviderConfig, error) { - confProFile := os.Getenv(constant.CONF_PROVIDER_FILE_PATH) - if len(confProFile) == 0 { - return nil, nil - } - restProviderConfig := &rest.RestProviderConfig{} - err := yaml.UnmarshalYMLConfig(confProFile, restProviderConfig) - if err != nil { - return nil, perrors.Errorf("[Rest Config] unmarshal Provider RestYmlConfig error %v", perrors.WithStack(err)) - - } - return restProviderConfig, nil -} - -func GetDefaultConfigReader() config_reader.RestConfigReader { - if defaultConfigReader == nil { - defaultConfigReader = NewDefaultConfigReader() - } - return defaultConfigReader -} From 4c50edc98b2b85939e3de9e915f6c72e954639a4 Mon Sep 17 00:00:00 2001 From: fangyincheng Date: Sun, 15 Mar 2020 16:41:06 +0800 Subject: [PATCH 19/22] Fix: resolve travis --- config/consumer_config.go | 2 +- config/provider_config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/consumer_config.go b/config/consumer_config.go index 21034e0dd5..868aa31fd4 100644 --- a/config/consumer_config.go +++ b/config/consumer_config.go @@ -62,7 +62,7 @@ type ConsumerConfig struct { ProtocolConf interface{} `yaml:"protocol_conf" json:"protocol_conf,omitempty" property:"protocol_conf"` FilterConf interface{} `yaml:"filter_conf" json:"filter_conf,omitempty" property:"filter_conf" ` ShutdownConfig *ShutdownConfig `yaml:"shutdown_conf" json:"shutdown_conf,omitempty" property:"shutdown_conf" ` - ConfigType string `default:"default" yaml:"config_type" json:"config_type,omitempty" property:"config_type"` + ConfigType string `yaml:"config_type" json:"config_type,omitempty" property:"config_type"` } // UnmarshalYAML ... diff --git a/config/provider_config.go b/config/provider_config.go index a45228f0a4..765e913933 100644 --- a/config/provider_config.go +++ b/config/provider_config.go @@ -53,7 +53,7 @@ type ProviderConfig struct { ProtocolConf interface{} `yaml:"protocol_conf" json:"protocol_conf,omitempty" property:"protocol_conf" ` FilterConf interface{} `yaml:"filter_conf" json:"filter_conf,omitempty" property:"filter_conf" ` ShutdownConfig *ShutdownConfig `yaml:"shutdown_conf" json:"shutdown_conf,omitempty" property:"shutdown_conf" ` - ConfigType string `default:"default" yaml:"config_type" json:"config_type,omitempty" property:"config_type"` + ConfigType string `yaml:"config_type" json:"config_type,omitempty" property:"config_type"` } // UnmarshalYAML ... From 1e9392a9fb0829e276308e1f9d3236b5b30f9359 Mon Sep 17 00:00:00 2001 From: fangyincheng Date: Sun, 15 Mar 2020 19:56:52 +0800 Subject: [PATCH 20/22] Imp: improve config reader --- common/extension/confit_reader.go | 11 ++++++++ config/base_config.go | 4 ++- config/config_loader.go | 25 +++++++++++++++++++ config/consumer_config.go | 18 ++----------- config/provider_config.go | 19 +++----------- .../rest/config/reader/rest_config_reader.go | 1 + .../reader/testdata/consumer_config.yml | 3 +++ .../reader/testdata/provider_config.yml | 4 +++ protocol/rest/rest_protocol.go | 1 + 9 files changed, 53 insertions(+), 33 deletions(-) diff --git a/common/extension/confit_reader.go b/common/extension/confit_reader.go index fe7124cf42..a3b8927ba1 100644 --- a/common/extension/confit_reader.go +++ b/common/extension/confit_reader.go @@ -23,6 +23,7 @@ import ( var ( configReaders = make(map[string]func() interfaces.ConfigReader) + defaults = make(map[string]string) ) // SetConfigReaders set a creator of config reader. @@ -37,3 +38,13 @@ func GetConfigReaders(name string) interfaces.ConfigReader { } return configReaders[name]() } + +// SetDefaultConfitReader set {name} to default config reader for {module} +func SetDefaultConfitReader(module, name string) { + defaults[module] = name +} + +// GetDefaultConfitReader +func GetDefaultConfitReader() map[string]string { + return defaults +} diff --git a/config/base_config.go b/config/base_config.go index 787297c185..93c0ce6a66 100644 --- a/config/base_config.go +++ b/config/base_config.go @@ -18,6 +18,7 @@ package config import ( + "bytes" "reflect" "strconv" "strings" @@ -47,6 +48,8 @@ type BaseConfig struct { fatherConfig interface{} MetricConfig *MetricConfig `yaml:"metrics" json:"metrics,omitempty"` + + fileStream *bytes.Buffer } // startConfigCenter will start the config center. @@ -361,5 +364,4 @@ func initializeStruct(t reflect.Type, v reflect.Value) { } } - } diff --git a/config/config_loader.go b/config/config_loader.go index 14acc7bd73..8b6f43f249 100644 --- a/config/config_loader.go +++ b/config/config_loader.go @@ -19,6 +19,8 @@ package config import ( "fmt" + "github.com/apache/dubbo-go/common/extension" + perrors "github.com/pkg/errors" "log" "os" "time" @@ -79,6 +81,29 @@ func checkApplicationName(config *ApplicationConfig) { // Load Dubbo Init func Load() { + // init other consumer config + conConfigType := consumerConfig.ConfigType + for key, value := range extension.GetDefaultConfitReader() { + if v, ok := conConfigType[key]; ok { + value = v + } + if err := extension.GetConfigReaders(value).ReadConsumerConfig(consumerConfig.fileStream); err != nil { + logger.Errorf("ReadConsumerConfig error: %#v for %s", perrors.WithStack(err), value) + } + } + + // init other provider config + proConfigType := providerConfig.ConfigType + for key, value := range extension.GetDefaultConfitReader() { + if v, ok := proConfigType[key]; ok { + value = v + } + + if err := extension.GetConfigReaders(value).ReadProviderConfig(providerConfig.fileStream); err != nil { + logger.Errorf("ReadProviderConfig error: %#v for %s", perrors.WithStack(err), value) + } + } + // init router if confRouterFile != "" { if errPro := RouterInit(confRouterFile); errPro != nil { diff --git a/config/consumer_config.go b/config/consumer_config.go index 868aa31fd4..1fa68415bf 100644 --- a/config/consumer_config.go +++ b/config/consumer_config.go @@ -19,9 +19,6 @@ package config import ( "bytes" - "fmt" - "github.com/apache/dubbo-go/common/extension" - "strings" "time" ) @@ -62,7 +59,7 @@ type ConsumerConfig struct { ProtocolConf interface{} `yaml:"protocol_conf" json:"protocol_conf,omitempty" property:"protocol_conf"` FilterConf interface{} `yaml:"filter_conf" json:"filter_conf,omitempty" property:"filter_conf" ` ShutdownConfig *ShutdownConfig `yaml:"shutdown_conf" json:"shutdown_conf,omitempty" property:"shutdown_conf" ` - ConfigType string `yaml:"config_type" json:"config_type,omitempty" property:"config_type"` + ConfigType map[string]string `yaml:"config_type" json:"config_type,omitempty" property:"config_type"` } // UnmarshalYAML ... @@ -97,6 +94,7 @@ func ConsumerInit(confConFile string) error { if err != nil { return perrors.Errorf("unmarshalYmlConfig error %v", perrors.WithStack(err)) } + consumerConfig.fileStream = bytes.NewBuffer(fileStream) //set method interfaceId & interfaceName for k, v := range consumerConfig.References { //set id for reference @@ -121,18 +119,6 @@ func ConsumerInit(confConFile string) error { } logger.Debugf("consumer config{%#v}\n", consumerConfig) - // init other consumer config - conConfigType := consumerConfig.ConfigType - if len(conConfigType) > 0 { - for _, t := range strings.Split(conConfigType, ",") { - if len(t) > 0 { - if err = extension.GetConfigReaders(t).ReadConsumerConfig(bytes.NewBuffer(fileStream)); err != nil { - return perrors.New(fmt.Sprintf("ReadConsumerConfig error: %v for %s", perrors.WithStack(err), t)) - } - } - } - } - return nil } diff --git a/config/provider_config.go b/config/provider_config.go index 765e913933..14b77cafb3 100644 --- a/config/provider_config.go +++ b/config/provider_config.go @@ -19,8 +19,6 @@ package config import ( "bytes" - "fmt" - "strings" ) import ( @@ -30,7 +28,6 @@ import ( import ( "github.com/apache/dubbo-go/common/constant" - "github.com/apache/dubbo-go/common/extension" "github.com/apache/dubbo-go/common/logger" "github.com/apache/dubbo-go/common/yaml" ) @@ -53,7 +50,7 @@ type ProviderConfig struct { ProtocolConf interface{} `yaml:"protocol_conf" json:"protocol_conf,omitempty" property:"protocol_conf" ` FilterConf interface{} `yaml:"filter_conf" json:"filter_conf,omitempty" property:"filter_conf" ` ShutdownConfig *ShutdownConfig `yaml:"shutdown_conf" json:"shutdown_conf,omitempty" property:"shutdown_conf" ` - ConfigType string `yaml:"config_type" json:"config_type,omitempty" property:"config_type"` + ConfigType map[string]string `yaml:"config_type" json:"config_type,omitempty" property:"config_type"` } // UnmarshalYAML ... @@ -88,6 +85,8 @@ func ProviderInit(confProFile string) error { if err != nil { return perrors.Errorf("unmarshalYmlConfig error %v", perrors.WithStack(err)) } + + providerConfig.fileStream = bytes.NewBuffer(fileStream) //set method interfaceId & interfaceName for k, v := range providerConfig.Services { //set id for reference @@ -99,18 +98,6 @@ func ProviderInit(confProFile string) error { logger.Debugf("provider config{%#v}\n", providerConfig) - // init other provider config - proConfigType := providerConfig.ConfigType - if len(proConfigType) > 0 { - for _, t := range strings.Split(proConfigType, ",") { - if len(t) > 0 { - if err = extension.GetConfigReaders(t).ReadProviderConfig(bytes.NewBuffer(fileStream)); err != nil { - return perrors.New(fmt.Sprintf("ReadProviderConfig error: %v for %s", perrors.WithStack(err), t)) - } - } - } - } - return nil } diff --git a/protocol/rest/config/reader/rest_config_reader.go b/protocol/rest/config/reader/rest_config_reader.go index ff19e97e2e..0691096bf9 100644 --- a/protocol/rest/config/reader/rest_config_reader.go +++ b/protocol/rest/config/reader/rest_config_reader.go @@ -40,6 +40,7 @@ const REST = "rest" func init() { extension.SetConfigReaders(REST, NewRestConfigReader) + extension.SetDefaultConfitReader(REST, REST) } type RestConfigReader struct { diff --git a/protocol/rest/config/reader/testdata/consumer_config.yml b/protocol/rest/config/reader/testdata/consumer_config.yml index 4cbb86fd54..27d7fdafef 100644 --- a/protocol/rest/config/reader/testdata/consumer_config.yml +++ b/protocol/rest/config/reader/testdata/consumer_config.yml @@ -2,6 +2,9 @@ filter: "" +config_type: + rest: "rest" + # client request_timeout : "100ms" # connect timeout diff --git a/protocol/rest/config/reader/testdata/provider_config.yml b/protocol/rest/config/reader/testdata/provider_config.yml index 7ca544491a..71d056e727 100644 --- a/protocol/rest/config/reader/testdata/provider_config.yml +++ b/protocol/rest/config/reader/testdata/provider_config.yml @@ -1,6 +1,10 @@ # dubbo server yaml configure file filter: "" + +config_type: + rest: "rest" + # application config application: organization : "ikurento.com" diff --git a/protocol/rest/rest_protocol.go b/protocol/rest/rest_protocol.go index 8801697534..47ecb6093b 100644 --- a/protocol/rest/rest_protocol.go +++ b/protocol/rest/rest_protocol.go @@ -33,6 +33,7 @@ import ( "github.com/apache/dubbo-go/protocol/rest/client" _ "github.com/apache/dubbo-go/protocol/rest/client/client_impl" rest_config "github.com/apache/dubbo-go/protocol/rest/config" + _ "github.com/apache/dubbo-go/protocol/rest/config/reader" "github.com/apache/dubbo-go/protocol/rest/server" _ "github.com/apache/dubbo-go/protocol/rest/server/server_impl" ) From 17c29b5c364bc114db02c2f4e603ba54397a35b9 Mon Sep 17 00:00:00 2001 From: fangyincheng Date: Sun, 15 Mar 2020 20:13:46 +0800 Subject: [PATCH 21/22] Fix: fix nil point --- config/config_loader.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/config/config_loader.go b/config/config_loader.go index 8b6f43f249..98bd9aae48 100644 --- a/config/config_loader.go +++ b/config/config_loader.go @@ -84,8 +84,10 @@ func Load() { // init other consumer config conConfigType := consumerConfig.ConfigType for key, value := range extension.GetDefaultConfitReader() { - if v, ok := conConfigType[key]; ok { - value = v + if conConfigType == nil { + if v, ok := conConfigType[key]; ok { + value = v + } } if err := extension.GetConfigReaders(value).ReadConsumerConfig(consumerConfig.fileStream); err != nil { logger.Errorf("ReadConsumerConfig error: %#v for %s", perrors.WithStack(err), value) @@ -95,10 +97,11 @@ func Load() { // init other provider config proConfigType := providerConfig.ConfigType for key, value := range extension.GetDefaultConfitReader() { - if v, ok := proConfigType[key]; ok { - value = v + if proConfigType != nil { + if v, ok := proConfigType[key]; ok { + value = v + } } - if err := extension.GetConfigReaders(value).ReadProviderConfig(providerConfig.fileStream); err != nil { logger.Errorf("ReadProviderConfig error: %#v for %s", perrors.WithStack(err), value) } From d7cc3afac49da88a9e7fa561651e7a37121ab263 Mon Sep 17 00:00:00 2001 From: fangyincheng Date: Sun, 15 Mar 2020 21:07:09 +0800 Subject: [PATCH 22/22] Fix: reviews --- config/config_loader.go | 52 ++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/config/config_loader.go b/config/config_loader.go index 98bd9aae48..6bf72680ac 100644 --- a/config/config_loader.go +++ b/config/config_loader.go @@ -81,32 +81,6 @@ func checkApplicationName(config *ApplicationConfig) { // Load Dubbo Init func Load() { - // init other consumer config - conConfigType := consumerConfig.ConfigType - for key, value := range extension.GetDefaultConfitReader() { - if conConfigType == nil { - if v, ok := conConfigType[key]; ok { - value = v - } - } - if err := extension.GetConfigReaders(value).ReadConsumerConfig(consumerConfig.fileStream); err != nil { - logger.Errorf("ReadConsumerConfig error: %#v for %s", perrors.WithStack(err), value) - } - } - - // init other provider config - proConfigType := providerConfig.ConfigType - for key, value := range extension.GetDefaultConfitReader() { - if proConfigType != nil { - if v, ok := proConfigType[key]; ok { - value = v - } - } - if err := extension.GetConfigReaders(value).ReadProviderConfig(providerConfig.fileStream); err != nil { - logger.Errorf("ReadProviderConfig error: %#v for %s", perrors.WithStack(err), value) - } - } - // init router if confRouterFile != "" { if errPro := RouterInit(confRouterFile); errPro != nil { @@ -118,6 +92,19 @@ func Load() { if consumerConfig == nil { logger.Warnf("consumerConfig is nil!") } else { + // init other consumer config + conConfigType := consumerConfig.ConfigType + for key, value := range extension.GetDefaultConfitReader() { + if conConfigType == nil { + if v, ok := conConfigType[key]; ok { + value = v + } + } + if err := extension.GetConfigReaders(value).ReadConsumerConfig(consumerConfig.fileStream); err != nil { + logger.Errorf("ReadConsumerConfig error: %#v for %s", perrors.WithStack(err), value) + } + } + metricConfig = consumerConfig.MetricConfig applicationConfig = consumerConfig.ApplicationConfig @@ -178,6 +165,19 @@ func Load() { if providerConfig == nil { logger.Warnf("providerConfig is nil!") } else { + // init other provider config + proConfigType := providerConfig.ConfigType + for key, value := range extension.GetDefaultConfitReader() { + if proConfigType != nil { + if v, ok := proConfigType[key]; ok { + value = v + } + } + if err := extension.GetConfigReaders(value).ReadProviderConfig(providerConfig.fileStream); err != nil { + logger.Errorf("ReadProviderConfig error: %#v for %s", perrors.WithStack(err), value) + } + } + // so, you should know that the consumer's config will be override metricConfig = providerConfig.MetricConfig applicationConfig = providerConfig.ApplicationConfig