From d31759858daf199545f793c31ccb64fc71ef467c Mon Sep 17 00:00:00 2001 From: weekface Date: Tue, 23 Oct 2018 11:35:04 +0800 Subject: [PATCH 01/12] stash --- cmd/scheduler/main.go | 68 ++++++++++++++++ pkg/scheduler/predicates/predicate.go | 30 +++++++ pkg/scheduler/scheduler.go | 76 +++++++++++++++++ pkg/scheduler/server/mux.go | 112 ++++++++++++++++++++++++++ pkg/util/util.go | 13 +++ 5 files changed, 299 insertions(+) create mode 100644 cmd/scheduler/main.go create mode 100644 pkg/scheduler/predicates/predicate.go create mode 100644 pkg/scheduler/scheduler.go create mode 100644 pkg/scheduler/server/mux.go diff --git a/cmd/scheduler/main.go b/cmd/scheduler/main.go new file mode 100644 index 00000000000..78bbd475022 --- /dev/null +++ b/cmd/scheduler/main.go @@ -0,0 +1,68 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "net/http" + _ "net/http/pprof" + "os" + + "time" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/scheduler/server" + "github.com/pingcap/tidb-operator/version" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/util/logs" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +var ( + printVersion bool + port int +) + +func init() { + flag.BoolVar(&printVersion, "V", false, "Show version and quit") + flag.BoolVar(&printVersion, "version", false, "Show version and quit") + flag.IntVar(&port, "port", 10262, "The port that the tidb scheduler's http service runs on (default 10262)") + flag.Parse() +} + +func main() { + if printVersion { + version.PrintVersionInfo() + os.Exit(0) + } + version.LogVersionInfo() + + logs.InitLogs() + defer logs.FlushLogs() + + cfg, err := rest.InClusterConfig() + if err != nil { + glog.Fatalf("failed to get config: %v", err) + } + kubeCli, err := kubernetes.NewForConfig(cfg) + if err != nil { + glog.Fatalf("failed to get kubernetes Clientset: %v", err) + } + + go wait.Forever(func() { + server.StartServer(kubeCli, port) + }, 5*time.Second) + glog.Fatal(http.ListenAndServe(":6060", nil)) +} diff --git a/pkg/scheduler/predicates/predicate.go b/pkg/scheduler/predicates/predicate.go new file mode 100644 index 00000000000..8d22016e538 --- /dev/null +++ b/pkg/scheduler/predicates/predicate.go @@ -0,0 +1,30 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package predicates + +import ( + apiv1 "k8s.io/api/core/v1" +) + +// Predicate is an interface as extender-implemented predicate functions +type Predicate interface { + // Name return the predicate name + Name() string + + // Filter function receives a *volume.TiDBVolume, an *apiv1.Pod and an *apiv1.PersistentVolumeClaim, + // should remove or add a Priority to every volumes, + // and return whether it is fusing after this predicate, and an error. + // Implementations must treat the *apiv1.Pod and *apiv1.PersistentVolumeClaim parameter as read-only and not modify it. + Filter(*apiv1.Pod) (bool, error) +} diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go new file mode 100644 index 00000000000..03f8c0ade97 --- /dev/null +++ b/pkg/scheduler/scheduler.go @@ -0,0 +1,76 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package scheduler + +import ( + "k8s.io/client-go/kubernetes" + schedulerapiv1 "k8s.io/kubernetes/pkg/scheduler/api/v1" +) + +// Scheduler is an interface for external processes to influence scheduling +// decisions made by kubernetes. This is typically needed for resources not directly +// managed by kubernetes. +type Scheduler interface { + // Filter based on extender-implemented predicate functions. The filtered list is + // expected to be a subset of the supplied list. + Filter(*schedulerapiv1.ExtenderArgs) (*schedulerapiv1.ExtenderFilterResult, error) + + // Prioritize based on extender-implemented priority functions. The returned scores & weight + // are used to compute the weighted score for an extender. The weighted scores are added to + // the scores computed by kubernetes scheduler. The total scores are used to do the host selection. + Priority(*schedulerapiv1.ExtenderArgs) (schedulerapiv1.HostPriorityList, error) +} + +type scheduler struct { + kubeCli kubernetes.Interface + // predicates []predicates.Predicate +} + +// NewScheduler returns a Scheduler +func NewScheduler(kubeCli kubernetes.Interface) Scheduler { + return &scheduler{ + kubeCli: kubeCli, + // predicates: []predicates.Predicate{ + // highavailability.NewHighAvailability(10, kubeCli), + //}, + } +} + +// Filter select a node from *schedulerapiv1.ExtenderArgs.Nodes when this is a pd or tikv pod +// else return the original nodes. +func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.ExtenderFilterResult, error) { + pod := &args.Pod + ns := pod.GetNamespace() + podName := pod.GetName() + kubeNodes := args.Nodes.Items + + return nil, nil +} + +// We didn't pass `prioritizeVerb` to kubernetes scheduler extender's config file, this method will not be called. +func (s *scheduler) Priority(args *schedulerapiv1.ExtenderArgs) (schedulerapiv1.HostPriorityList, error) { + result := schedulerapiv1.HostPriorityList{} + + // avoid index out of range panic + if len(args.Nodes.Items) > 0 { + result = append(result, schedulerapiv1.HostPriority{ + Host: args.Nodes.Items[0].Name, + Score: 0, + }) + } + + return result, nil +} + +var _ Scheduler = &scheduler{} diff --git a/pkg/scheduler/server/mux.go b/pkg/scheduler/server/mux.go new file mode 100644 index 00000000000..3ae71157bc6 --- /dev/null +++ b/pkg/scheduler/server/mux.go @@ -0,0 +1,112 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "fmt" + "net/http" + "sync" + + "github.com/pingcap/tidb-operator/pkg/scheduler" + + restful "github.com/emicklei/go-restful" + "github.com/golang/glog" + "k8s.io/client-go/kubernetes" + schedulerapiv1 "k8s.io/kubernetes/pkg/scheduler/api/v1" +) + +var ( + errFailToRead = restful.NewError(http.StatusBadRequest, "unable to read request body") + errFailToWrite = restful.NewError(http.StatusInternalServerError, "unable to write response") +) + +type server struct { + scheduler scheduler.Scheduler + // All scheduling should be serialized + lock sync.Mutex +} + +// StartServer start a kubernetes scheduler extender http apiserver +func StartServer(kubeCli kubernetes.Interface, port int) { + s := scheduler.NewScheduler(kubeCli) + svr := &server{scheduler: s} + + ws := new(restful.WebService) + ws. + Path("/scheduler"). + Consumes(restful.MIME_JSON). + Produces(restful.MIME_JSON) + + ws.Route(ws.POST("/filter").To(svr.filterNode). + Doc("filter nodes"). + Operation("filterNodes"). + Writes(schedulerapiv1.ExtenderFilterResult{})) + + ws.Route(ws.POST("/prioritize").To(svr.prioritizeNode). + Doc("prioritize nodes"). + Operation("prioritizeNodes"). + Writes(schedulerapiv1.HostPriorityList{})) + restful.Add(ws) + + glog.Infof("start scheduler extender server, listening on 0.0.0.0:%d", port) + glog.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil)) +} + +func (svr *server) filterNode(req *restful.Request, resp *restful.Response) { + svr.lock.Lock() + defer svr.lock.Unlock() + + args := &schedulerapiv1.ExtenderArgs{} + if err := req.ReadEntity(args); err != nil { + errorResponse(resp, errFailToRead) + return + } + + filterResult, err := svr.scheduler.Filter(args) + if err != nil { + errorResponse(resp, restful.NewError(http.StatusInternalServerError, + fmt.Sprintf("unable to filter nodes: %v", err))) + return + } + + if err := resp.WriteEntity(filterResult); err != nil { + errorResponse(resp, errFailToWrite) + } +} + +func (svr *server) prioritizeNode(req *restful.Request, resp *restful.Response) { + args := &schedulerapiv1.ExtenderArgs{} + if err := req.ReadEntity(args); err != nil { + errorResponse(resp, errFailToRead) + return + } + + priorityResult, err := svr.scheduler.Priority(args) + if err != nil { + errorResponse(resp, restful.NewError(http.StatusInternalServerError, + fmt.Sprintf("unable to priority nodes: %v", err))) + return + } + + if err := resp.WriteEntity(priorityResult); err != nil { + errorResponse(resp, errFailToWrite) + } +} + +func errorResponse(resp *restful.Response, svcErr restful.ServiceError) { + glog.Error(svcErr.Message) + if writeErr := resp.WriteServiceError(svcErr.Code, svcErr); writeErr != nil { + glog.Errorf("unable to write error: %v", writeErr) + } +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 64e8a32552e..af8ab47259d 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1,3 +1,16 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + package util import ( From 449176ebcaa90b37be63156eb72a27fee5266a01 Mon Sep 17 00:00:00 2001 From: weekface Date: Thu, 25 Oct 2018 11:11:13 +0800 Subject: [PATCH 02/12] scheduler: add extended scheduler to operator --- Makefile | 5 +- .../tidb-cluster/templates/tidb-cluster.yaml | 1 + charts/tidb-cluster/values.yaml | 6 +- .../templates/scheduler-deployment.yaml | 64 ++++ .../templates/scheduler-policy-configmap.yaml | 50 +++ .../templates/scheduler-rbac.yaml | 125 +++++++ charts/tidb-operator/values.yaml | 22 ++ cmd/scheduler/main.go | 7 +- .../tidb-cluster-values.yaml | 74 ++++ .../tidb-operator-values.yaml | 34 +- images/tidb-operator/Dockerfile | 1 + pkg/apis/pingcap.com/v1alpha1/types.go | 1 + pkg/manager/member/pd_failover.go | 18 +- pkg/manager/member/pd_member_manager.go | 1 + pkg/manager/member/tidb_member_manager.go | 1 + pkg/manager/member/tikv_member_manager.go | 1 + pkg/scheduler/predicates/ha.go | 132 ++++++++ pkg/scheduler/predicates/ha_test.go | 317 ++++++++++++++++++ pkg/scheduler/predicates/predicate.go | 20 +- pkg/scheduler/scheduler.go | 55 ++- pkg/scheduler/server/mux.go | 12 +- pkg/util/util.go | 10 + 22 files changed, 915 insertions(+), 42 deletions(-) create mode 100644 charts/tidb-operator/templates/scheduler-deployment.yaml create mode 100644 charts/tidb-operator/templates/scheduler-policy-configmap.yaml create mode 100644 charts/tidb-operator/templates/scheduler-rbac.yaml create mode 100644 pkg/scheduler/predicates/ha.go create mode 100644 pkg/scheduler/predicates/ha_test.go diff --git a/Makefile b/Makefile index bd3912b2403..53aaa0e2a10 100644 --- a/Makefile +++ b/Makefile @@ -21,11 +21,14 @@ docker-push: docker docker: build docker build --tag "${DOCKER_REGISTRY}/pingcap/tidb-operator:latest" images/tidb-operator -build: controller-manager +build: controller-manager scheduler controller-manager: $(GO) build -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-controller-manager cmd/controller-manager/main.go +scheduler: + $(GO) build -ldflags '$(LDFLAGS)' -o images/tidb-operator/bin/tidb-scheduler cmd/scheduler/main.go + e2e-docker-push: e2e-docker docker push "${DOCKER_REGISTRY}/pingcap/tidb-operator-e2e:latest" diff --git a/charts/tidb-cluster/templates/tidb-cluster.yaml b/charts/tidb-cluster/templates/tidb-cluster.yaml index 6afbb2c0203..1632d8883be 100644 --- a/charts/tidb-cluster/templates/tidb-cluster.yaml +++ b/charts/tidb-cluster/templates/tidb-cluster.yaml @@ -13,6 +13,7 @@ spec: timezone: {{ .Values.timezone | default "UTC" }} services: {{ toYaml .Values.services | indent 4 }} + schedulerName: {{ .Values.schedulerName | default "default-scheduler" }} pd: replicas: {{ .Values.pd.replicas }} image: {{ .Values.pd.image }} diff --git a/charts/tidb-cluster/values.yaml b/charts/tidb-cluster/values.yaml index 9fe111e1776..da62edf46ab 100644 --- a/charts/tidb-cluster/values.yaml +++ b/charts/tidb-cluster/values.yaml @@ -11,6 +11,9 @@ rbac: # if multiple clusters are deployed in the same namespace. clusterName: demo +# schedulerName must be same with charts/tidb-operator/values#scheduler.schedulerName +schedulerName: tidb-scheduler + # timezone is the default system timzone for TiDB timezone: UTC @@ -131,11 +134,11 @@ tidb: # cloud.google.com/load-balancer-type: Internal monitor: + create: true # Also see rbac.create # If you set rbac.create to false, you need to provide a value here. # If you set rbac.create to true, you should leave this empty. serviceAccount: - create: true persistent: false storageClassName: local-storage storage: 10Gi @@ -179,7 +182,6 @@ monitor: # operator: Equal # value: tidb # effect: "NoSchedule" - # GCP MarketPlace integration ubbagent: {} diff --git a/charts/tidb-operator/templates/scheduler-deployment.yaml b/charts/tidb-operator/templates/scheduler-deployment.yaml new file mode 100644 index 00000000000..07b15c336e4 --- /dev/null +++ b/charts/tidb-operator/templates/scheduler-deployment.yaml @@ -0,0 +1,64 @@ +{{- $defaultHyperkubeImage := "quay.io/coreos/hyperkube:v1.10.4_coreos.0" -}} +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: tidb-scheduler + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: scheduler + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +spec: + replicas: {{ .Values.scheduler.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: scheduler + template: + metadata: + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: scheduler + spec: + {{- if .Values.scheduler.serviceAccount }} + serviceAccount: {{ .Values.scheduler.serviceAccount }} + {{- end }} + containers: + - name: tidb-scheduler + image: {{ .Values.operatorImage }} + resources: +{{ toYaml .Values.scheduler.resources | indent 12 }} + command: + - /usr/local/bin/tidb-scheduler + - -v={{ .Values.scheduler.logLevel }} + - -port=10262 + - -pd-replicas={{ .Values.scheduler.pdReplicas | default 3 }} + - -tikv-replicas={{ .Values.scheduler.tikvReplicas | default 3 }} + - name: kube-scheduler + {{- if .Values.scheduler.kubeSchedulerImage }} + image: {{ .Values.scheduler.kubeSchedulerImage }} + {{- else if .Values.scheduler.hyperkubeImage }} + image: {{ .Values.scheduler.hyperkubeImage }} + {{- else }} + image: {{ $defaultHyperkubeImage }} + {{- end }} + resources: +{{ toYaml .Values.scheduler.resources | indent 12 }} + command: + {{- if .Values.scheduler.kubeSchedulerImage }} + - kube-scheduler + {{- else }} + - /hyperkube + - scheduler + {{- end }} + - --port=10261 + - --leader-elect=true + - --lock-object-name=tidb-scheduler + - --lock-object-namespace={{ .Release.Namespace }} + - --scheduler-name={{ .Values.scheduler.schedulerName }} + - --v={{ .Values.scheduler.logLevel }} + - --policy-configmap=tidb-scheduler-policy + - --policy-configmap-namespace={{ .Release.Namespace }} diff --git a/charts/tidb-operator/templates/scheduler-policy-configmap.yaml b/charts/tidb-operator/templates/scheduler-policy-configmap.yaml new file mode 100644 index 00000000000..1524ca36a21 --- /dev/null +++ b/charts/tidb-operator/templates/scheduler-policy-configmap.yaml @@ -0,0 +1,50 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: tidb-scheduler-policy + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: scheduler + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +data: + policy.cfg: |- + { + "kind" : "Policy", + "apiVersion" : "v1", + "predicates": [ + {"name": "MatchInterPodAffinity"}, + {"name": "CheckVolumeBinding"}, + {"name": "CheckNodeCondition"}, + {"name": "GeneralPredicates"}, + {"name": "HostName"}, + {"name": "PodFitsHostPorts"}, + {"name": "MatchNodeSelector"}, + {"name": "PodFitsResources"}, + {"name": "NoDiskConflict"}, + {"name": "PodToleratesNodeTaints"}, + {"name": "CheckNodeMemoryPressure"}, + {"name": "CheckNodeDiskPressure"} + ], + "priorities": [ + {"name": "EqualPriority", "weight": 1}, + {"name": "ImageLocalityPriority", "weight": 1}, + {"name": "LeastRequestedPriority", "weight": 1}, + {"name": "BalancedResourceAllocation", "weight": 1}, + {"name": "SelectorSpreadPriority", "weight": 1}, + {"name": "NodePreferAvoidPodsPriority", "weight": 1}, + {"name": "NodeAffinityPriority", "weight": 1}, + {"name": "TaintTolerationPriority", "weight": 1}, + {"name": "MostRequestedPriority", "weight": 1} + ], + "extenders": [ + { + "urlPrefix": "http://127.0.0.1:10262/scheduler", + "filterVerb": "filter", + "weight": 1, + "httpTimeout": 30000000000, + "enableHttps": false + } + ] + } diff --git a/charts/tidb-operator/templates/scheduler-rbac.yaml b/charts/tidb-operator/templates/scheduler-rbac.yaml new file mode 100644 index 00000000000..bdb23813651 --- /dev/null +++ b/charts/tidb-operator/templates/scheduler-rbac.yaml @@ -0,0 +1,125 @@ +{{- if .Values.rbac.create }} +kind: ServiceAccount +apiVersion: v1 +metadata: + name: {{ .Values.scheduler.serviceAccount }} + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: scheduler + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: {{ .Release.Name }}:tidb-scheduler + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: scheduler + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +rules: +{{- if .Values.clusterScoped }} +- apiGroups: [""] + resources: ["pods", "services", "configmaps", "replicationcontrollers", "persistentvolumeclaims", "endpoints"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["pods/binding"] + verbs: ["create"] +- apiGroups: [""] + resources: ["endpoints", "events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +- apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["extensions"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +{{- end }} +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: {{ .Release.Name }}:tidb-scheduler + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: scheduler + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +subjects: +- kind: ServiceAccount + name: {{ .Values.scheduler.serviceAccount }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Release.Name }}:tidb-scheduler + apiGroup: rbac.authorization.k8s.io +{{- if (not .Values.clusterScoped) }} +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: {{ .Release.Name }}:tidb-scheduler + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: scheduler + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +rules: +- apiGroups: [""] + resources: ["pods", "services", "configmaps", "replicationcontrollers", "persistentvolumeclaims", "endpoints"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["pods/binding"] + verbs: ["create"] +- apiGroups: [""] + resources: ["endpoints", "events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +- apiGroups: ["apps"] + resources: ["statefulsets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["extensions"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: {{ .Release.Name }}:tidb-scheduler + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: scheduler + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +subjects: +- kind: ServiceAccount + name: {{ .Values.scheduler.serviceAccount }} +roleRef: + kind: Role + name: {{ .Release.Name }}:tidb-scheduler + apiGroup: rbac.authorization.k8s.io +{{- end }} +{{- end }} diff --git a/charts/tidb-operator/values.yaml b/charts/tidb-operator/values.yaml index 07094481b3e..a5c2c5c645b 100644 --- a/charts/tidb-operator/values.yaml +++ b/charts/tidb-operator/values.yaml @@ -34,3 +34,25 @@ controllerManager: pdFailoverPeriod: 5m # tidb failover period default(5m) tidbFailoverPeriod: 5m + +scheduler: + # With rbac.create=false, the user is responsible for creating this account + # With rbac.create=true, this service account will be created + # Also see rbac.create and clusterScoped + serviceAccount: tidb-scheduler + logLevel: 2 + replicas: 1 + schedulerName: tidb-scheduler + resources: + limits: + cpu: 250m + memory: 150Mi + requests: + cpu: 80m + memory: 50Mi + # pd replicas + pdReplicas: 3 + # tikv replicas + tikvReplicas: 3 + hyperkubeImage: quay.io/coreos/hyperkube:v1.10.4_coreos.0 + # kubeSchedulerImage: diff --git a/cmd/scheduler/main.go b/cmd/scheduler/main.go index 78bbd475022..01a6687a015 100644 --- a/cmd/scheduler/main.go +++ b/cmd/scheduler/main.go @@ -18,7 +18,6 @@ import ( "net/http" _ "net/http/pprof" "os" - "time" "github.com/golang/glog" @@ -33,12 +32,16 @@ import ( var ( printVersion bool port int + pdReplicas int + tikvReplicas int ) func init() { flag.BoolVar(&printVersion, "V", false, "Show version and quit") flag.BoolVar(&printVersion, "version", false, "Show version and quit") flag.IntVar(&port, "port", 10262, "The port that the tidb scheduler's http service runs on (default 10262)") + flag.IntVar(&pdReplicas, "pd-replicas", 3, "The pd replicas (default 3)") + flag.IntVar(&tikvReplicas, "tikv-replicas", 3, "The tikv replicas (default 3)") flag.Parse() } @@ -62,7 +65,7 @@ func main() { } go wait.Forever(func() { - server.StartServer(kubeCli, port) + server.StartServer(kubeCli, port, int32(pdReplicas), int32(tikvReplicas)) }, 5*time.Second) glog.Fatal(http.ListenAndServe(":6060", nil)) } diff --git a/images/tidb-operator-e2e/tidb-cluster-values.yaml b/images/tidb-operator-e2e/tidb-cluster-values.yaml index 0ed3d2e3683..844082299ff 100644 --- a/images/tidb-operator-e2e/tidb-cluster-values.yaml +++ b/images/tidb-operator-e2e/tidb-cluster-values.yaml @@ -2,6 +2,8 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. +# Also see monitor.serviceAccount +# If you set rbac.create to false, you need to provide a value for monitor.serviceAccount rbac: create: true @@ -9,6 +11,9 @@ rbac: # if multiple clusters are deployed in the same namespace. clusterName: demo-cluster +# schedulerName must be same with charts/tidb-operator/values#scheduler.schedulerName +schedulerName: tidb-scheduler + # timezone is the default system timzone for TiDB timezone: UTC @@ -130,6 +135,10 @@ tidb: monitor: create: true + # Also see rbac.create + # If you set rbac.create to false, you need to provide a value here. + # If you set rbac.create to true, you should leave this empty. + serviceAccount: persistent: false storageClassName: local-storage storage: 10Gi @@ -173,6 +182,8 @@ monitor: # operator: Equal # value: tidb # effect: "NoSchedule" + # GCP MarketPlace integration + ubbagent: {} fullbackup: create: false @@ -207,6 +218,69 @@ fullbackup: # credentialsData: "" # bucket: "" +binlog: + pump: + create: false + replicas: 1 + image: pingcap/tidb-binlog:new + imagePullPolicy: IfNotPresent + logLevel: info + # storageClassName is a StorageClass provides a way for administrators to describe the "classes" of storage they offer. + # different classes might map to quality-of-service levels, or to backup policies, + # or to arbitrary policies determined by the cluster administrators. + # refer to https://kubernetes.io/docs/concepts/storage/storage-classes + storageClassName: local-storage + storage: 10Gi + # a integer value to control expiry date of the binlog data, indicates for how long (in days) the binlog data would be stored. + # must bigger than 0 + gc: 7 + # number of seconds between heartbeat ticks (in 2 seconds) + heartbeatInterval: 2 + + drainer: + create: false + image: pingcap/tidb-binlog:new + imagePullPolicy: IfNotPresent + logLevel: info + # storageClassName is a StorageClass provides a way for administrators to describe the "classes" of storage they offer. + # different classes might map to quality-of-service levels, or to backup policies, + # or to arbitrary policies determined by the cluster administrators. + # refer to https://kubernetes.io/docs/concepts/storage/storage-classes + storageClassName: local-storage + storage: 10Gi + # parallel worker count (default 1) + workerCount: 1 + # the interval time (in seconds) of detect pumps' status (default 10) + detectInterval: 10 + # disbale detect causality + disableDetect: false + # disable dispatching sqls that in one same binlog; if set true, work-count and txn-batch would be useless + disableDispatch: false + # # disable sync these schema + ignoreSchemas: "INFORMATION_SCHEMA,PERFORMANCE_SCHEMA,mysql,test" + # if drainer donesn't have checkpoint, use initial commitTS to initial checkpoint + initialCommitTs: 0 + # enable safe mode to make syncer reentrant + safeMode: false + # number of binlog events in a transaction batch (default 1) + txnBatch: 1 + # downstream storage, equal to --dest-db-type + # valid values are "mysql", "pb", "kafka" + destDBType: pb + mysql: {} + # host: "127.0.0.1" + # user: "root" + # password: "" + # port: 3306 + # # Time and size limits for flash batch write + # timeLimit: "30s" + # sizeLimit: "100000" + kafka: {} + # only need config one of zookeeper-addrs and kafka-addrs, will get kafka address if zookeeper-addrs is configed. + # zookeeperAddrs: "127.0.0.1:2181" + # kafkaAddrs: "127.0.0.1:9092" + # kafkaVersion: "0.8.2.0" + metaInstance: "{{ $labels.instance }}" metaType: "{{ $labels.type }}" metaValue: "{{ $value }}" diff --git a/images/tidb-operator-e2e/tidb-operator-values.yaml b/images/tidb-operator-e2e/tidb-operator-values.yaml index 8027798a26b..9a1575433ef 100644 --- a/images/tidb-operator-e2e/tidb-operator-values.yaml +++ b/images/tidb-operator-e2e/tidb-operator-values.yaml @@ -1,4 +1,10 @@ # Default values for tidb-operator + +# clusterScoped is whether tidb-operator should manage kubernetes cluster wide tidb clusters +# Also see rbac.create and controllerManager.serviceAccount +clusterScoped: true + +# Also see clusterScoped and controllerManager.serviceAccount rbac: create: true @@ -8,10 +14,10 @@ imagePullPolicy: Always defaultStorageClassName: local-storage -# clusterScoped is whether tidb-operator should manage kubernetes cluster wide tidb clusters -clusterScoped: true - controllerManager: + # With rbac.create=false, the user is responsible for creating this account + # With rbac.create=true, this service account will be created + # Also see rbac.create and clusterScoped serviceAccount: tidb-controller-manager logLevel: 2 replicas: 1 @@ -29,3 +35,25 @@ controllerManager: # tidb failover period default(5m) tidbFailoverPeriod: 5m +scheduler: + # With rbac.create=false, the user is responsible for creating this account + # With rbac.create=true, this service account will be created + # Also see rbac.create and clusterScoped + serviceAccount: tidb-scheduler + logLevel: 2 + replicas: 1 + schedulerName: tidb-scheduler + resources: + limits: + cpu: 250m + memory: 150Mi + requests: + cpu: 80m + memory: 50Mi + # pd replicas + pdReplicas: 3 + # tikv replicas + tikvReplicas: 3 + hyperkubeImage: quay.io/coreos/hyperkube:v1.10.4_coreos.0 + # kubeSchedulerImage: + diff --git a/images/tidb-operator/Dockerfile b/images/tidb-operator/Dockerfile index bf020d4d8a5..b31c13e4f9e 100644 --- a/images/tidb-operator/Dockerfile +++ b/images/tidb-operator/Dockerfile @@ -2,3 +2,4 @@ FROM alpine:3.5 RUN apk add tzdata --no-cache ADD bin/tidb-controller-manager /usr/local/bin/tidb-controller-manager +ADD bin/tidb-scheduler /usr/local/bin/tidb-scheduler diff --git a/pkg/apis/pingcap.com/v1alpha1/types.go b/pkg/apis/pingcap.com/v1alpha1/types.go index 9ec6abb9de8..49fc9dd8de4 100644 --- a/pkg/apis/pingcap.com/v1alpha1/types.go +++ b/pkg/apis/pingcap.com/v1alpha1/types.go @@ -88,6 +88,7 @@ type TidbClusterList struct { // TidbClusterSpec describes the attributes that a user creates on a tidb cluster type TidbClusterSpec struct { + SchedulerName string `json:"schedulerName,omitempty"` PD PDSpec `json:"pd,omitempty"` TiDB TiDBSpec `json:"tidb,omitempty"` TiKV TiKVSpec `json:"tikv,omitempty"` diff --git a/pkg/manager/member/pd_failover.go b/pkg/manager/member/pd_failover.go index 0507c203194..a8b952762b2 100644 --- a/pkg/manager/member/pd_failover.go +++ b/pkg/manager/member/pd_failover.go @@ -15,13 +15,12 @@ package member import ( "fmt" - "strconv" - "strings" "time" "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/util" "k8s.io/apimachinery/pkg/api/errors" corelisters "k8s.io/client-go/listers/core/v1" ) @@ -68,7 +67,7 @@ func (pf *pdFailover) cleanOrphanPods(tc *v1alpha1.TidbCluster) error { } podName := pdMember.PodName - ordinal, err := getOrdinalFromPodName(podName) + ordinal, err := util.GetOrdinalFromPodName(podName) if err != nil { return err } @@ -170,7 +169,7 @@ func (pf *pdFailover) Failover(tc *v1alpha1.TidbCluster) error { return err } - ordinal, err := getOrdinalFromPodName(failurePodName) + ordinal, err := util.GetOrdinalFromPodName(failurePodName) if err != nil { return err } @@ -226,7 +225,7 @@ func (pf *pdFailover) markThisMemberAsFailure(tc *v1alpha1.TidbCluster, pdMember ns := tc.GetNamespace() tcName := tc.GetName() podName := pdMember.Name - ordinal, err := getOrdinalFromPodName(podName) + ordinal, err := util.GetOrdinalFromPodName(podName) if err != nil { return err } @@ -254,15 +253,6 @@ func (pf *pdFailover) markThisMemberAsFailure(tc *v1alpha1.TidbCluster, pdMember return nil } -func getOrdinalFromPodName(podName string) (int32, error) { - ordinalStr := podName[strings.LastIndex(podName, "-")+1:] - ordinalInt, err := strconv.Atoi(ordinalStr) - if err != nil { - return int32(0), err - } - return int32(ordinalInt), nil -} - type fakePDFailover struct{} // NewFakePDFailover returns a fake Failover diff --git a/pkg/manager/member/pd_member_manager.go b/pkg/manager/member/pd_member_manager.go index dc6a94e90c3..0acae77ceae 100644 --- a/pkg/manager/member/pd_member_manager.go +++ b/pkg/manager/member/pd_member_manager.go @@ -510,6 +510,7 @@ func (pmm *pdMemberManager) getNewPDSetForTidbCluster(tc *v1alpha1.TidbCluster) Annotations: controller.AnnProm(2379), }, Spec: corev1.PodSpec{ + SchedulerName: tc.Spec.SchedulerName, Affinity: util.AffinityForNodeSelector( ns, tc.Spec.PD.NodeSelectorRequired, diff --git a/pkg/manager/member/tidb_member_manager.go b/pkg/manager/member/tidb_member_manager.go index 08e1d708f40..71af8b7cd8f 100644 --- a/pkg/manager/member/tidb_member_manager.go +++ b/pkg/manager/member/tidb_member_manager.go @@ -278,6 +278,7 @@ func (tmm *tidbMemberManager) getNewTiDBSetForTidbCluster(tc *v1alpha1.TidbClust Annotations: controller.AnnProm(10080), }, Spec: corev1.PodSpec{ + SchedulerName: tc.Spec.SchedulerName, Affinity: util.AffinityForNodeSelector( ns, tc.Spec.TiDB.NodeSelectorRequired, diff --git a/pkg/manager/member/tikv_member_manager.go b/pkg/manager/member/tikv_member_manager.go index 890cca673b3..17c07ec69b8 100644 --- a/pkg/manager/member/tikv_member_manager.go +++ b/pkg/manager/member/tikv_member_manager.go @@ -317,6 +317,7 @@ func (tkmm *tikvMemberManager) getNewSetForTidbCluster(tc *v1alpha1.TidbCluster) Annotations: controller.AnnProm(9091), }, Spec: corev1.PodSpec{ + SchedulerName: tc.Spec.SchedulerName, Affinity: util.AffinityForNodeSelector( ns, tc.Spec.TiKV.NodeSelectorRequired, diff --git a/pkg/scheduler/predicates/ha.go b/pkg/scheduler/predicates/ha.go new file mode 100644 index 00000000000..4872e6ca778 --- /dev/null +++ b/pkg/scheduler/predicates/ha.go @@ -0,0 +1,132 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package predicates + +import ( + "fmt" + + "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/util" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" +) + +type ha struct { + kubeCli kubernetes.Interface + podListFn func(ns, clusterName, component string) (*apiv1.PodList, error) + pdReplicas int32 + tikvReplicas int32 +} + +// NewHA returns a Predicate +func NewHA(kubeCli kubernetes.Interface, pdReplicas, tikvReplicas int32) Predicate { + h := &ha{ + kubeCli: kubeCli, + pdReplicas: pdReplicas, + tikvReplicas: tikvReplicas, + } + h.podListFn = h.realPodListFn + return h +} + +func (h *ha) Name() string { + return "HighAvailability" +} + +func (h *ha) Filter(clusterName string, pod *apiv1.Pod, nodes []apiv1.Node) ([]apiv1.Node, error) { + ns := pod.GetNamespace() + podName := pod.GetName() + + var component string + var exist bool + if component, exist = pod.Labels[label.ComponentLabelKey]; !exist { + return nil, fmt.Errorf("can't find component in pod labels: %s/%s", ns, podName) + } + podList, err := h.podListFn(ns, clusterName, component) + if err != nil { + return nil, err + } + + ordinal, err := util.GetOrdinalFromPodName(podName) + if err != nil { + return nil, err + } + + nodeMap := make(map[string][]string) + for _, node := range nodes { + nodeMap[node.GetName()] = make([]string, 0) + } + for _, pod := range podList.Items { + podName1 := pod.GetName() + nodeName := pod.Spec.NodeName + ordinal1, err := util.GetOrdinalFromPodName(podName1) + if err != nil { + return nil, err + } + + if ordinal1 < ordinal && nodeName == "" { + return nil, fmt.Errorf("waiting for pod: %s/%s to be scheduled", ns, podName1) + } + if nodeName == "" { + continue + } + + nodeMap[nodeName] = append(nodeMap[nodeName], podName1) + } + + var min int + var minInitialized bool + for _, podNameArr := range nodeMap { + count := len(podNameArr) + if !minInitialized { + minInitialized = true + min = count + } + if count < min { + min = count + } + } + minNodeNames := make([]string, 0) + for nodeName, podNameArr := range nodeMap { + if len(podNameArr) == min { + minNodeNames = append(minNodeNames, nodeName) + } + } + if len(minNodeNames) == 0 { + return nil, fmt.Errorf("no suitable node for pod: %s/%s", ns, podName) + } + + var replicas int32 + switch component { + case label.PDLabelVal: + replicas = h.pdReplicas + case label.TiKVLabelVal: + replicas = h.tikvReplicas + } + + if ordinal < replicas && min != 0 { + return nil, fmt.Errorf("the first %d pods can't scheduled to the same node", replicas) + } + + return getNodeFromNames(nodes, minNodeNames), nil +} + +func (h *ha) realPodListFn(ns, clusterName, component string) (*apiv1.PodList, error) { + selector := label.New().Cluster(clusterName).Component(component).Labels() + return h.kubeCli.CoreV1().Pods(ns).List(metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(selector).String(), + }) +} diff --git a/pkg/scheduler/predicates/ha_test.go b/pkg/scheduler/predicates/ha_test.go new file mode 100644 index 00000000000..b23618a68a7 --- /dev/null +++ b/pkg/scheduler/predicates/ha_test.go @@ -0,0 +1,317 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package predicates + +import ( + "errors" + "fmt" + "sort" + "strings" + "testing" + + . "github.com/onsi/gomega" + "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/label" + apiv1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestHAFilter(t *testing.T) { + g := NewGomegaWithT(t) + type testcase struct { + name string + ordinal int32 + podFn func(string, int32) *apiv1.Pod + nodesFn func() []apiv1.Node + podListFn func(string, string, string) (*apiv1.PodList, error) + pdReplicas int32 + tikvReplicas int32 + expectFn func([]apiv1.Node, error) + } + + testFn := func(test *testcase, t *testing.T) { + t.Log(test.name) + clusterName := "demo" + + pod := test.podFn(clusterName, test.ordinal) + nodes := test.nodesFn() + + ha := ha{podListFn: test.podListFn, pdReplicas: test.pdReplicas, tikvReplicas: test.tikvReplicas} + test.expectFn(ha.Filter(clusterName, pod, nodes)) + } + + tests := []testcase{ + { + name: "component key is empty", + ordinal: 0, + podFn: func(clusterName string, ordinal int32) *apiv1.Pod { + pod := newHAPDPod(clusterName, ordinal) + pod.Labels = nil + return pod + }, + nodesFn: threeNodes, + podListFn: podListFn(map[string][]int32{}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(strings.Contains(err.Error(), "can't find component in pod labels")).To(Equal(true)) + }, + }, + { + name: "pod list return error", + ordinal: 0, + podFn: newHAPDPod, + nodesFn: threeNodes, + podListFn: podListErr(), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(strings.Contains(err.Error(), "pod list error")).To(Equal(true)) + }, + }, + { + name: "get ordinal from podName error", + ordinal: 0, + podFn: func(clusterName string, ordinal int32) *apiv1.Pod { + pod := newHAPDPod(clusterName, ordinal) + pod.Name = "xxxx" + return pod + }, + nodesFn: threeNodes, + podListFn: podListFn(map[string][]int32{}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(strings.Contains(err.Error(), "strconv.Atoi: parsing")).To(Equal(true)) + }, + }, + { + name: "one pod, podName is wrong", + ordinal: 0, + podFn: newHAPDPod, + nodesFn: threeNodes, + podListFn: podNameWrongListFn(), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(strings.Contains(err.Error(), "strconv.Atoi: parsing")).To(Equal(true)) + }, + }, + { + name: "the lower oridnal is not scheduled", + ordinal: 1, + podFn: newHAPDPod, + nodesFn: threeNodes, + podListFn: podListFn(map[string][]int32{"": []int32{0}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(strings.Contains(err.Error(), "waiting for pod: default/demo-pd-0")).To(Equal(true)) + }, + }, + { + name: "no scheduled pods, three nodes, ordinal 0 should be scheduled to all nodes", + ordinal: 0, + podFn: newHAPDPod, + nodesFn: threeNodes, + podListFn: podListFn(map[string][]int32{}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).To(Equal(3)) + g.Expect(getSortedNodeNames(nodes)).To(Equal([]string{"kube-node-1", "kube-node-2", "kube-node-3"})) + }, + }, + { + name: "ordinal 0 is scheduled to kube-node-1, ordinal 1 should be scheduled to kube-node-2 or kube-node-3", + ordinal: 1, + podFn: newHAPDPod, + nodesFn: threeNodes, + podListFn: podListFn(map[string][]int32{"kube-node-1": []int32{0}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).To(Equal(2)) + g.Expect(getSortedNodeNames(nodes)).To(Equal([]string{"kube-node-2", "kube-node-3"})) + }, + }, + { + name: "ordinal 0 is scheduled to kube-node-2, ordinal 1 is kube-node-3, ordinal 2 should be scheduled to kube-node-1", + ordinal: 2, + podFn: newHAPDPod, + nodesFn: threeNodes, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).To(Equal(1)) + g.Expect(getSortedNodeNames(nodes)).To(Equal([]string{"kube-node-1"})) + }, + }, + { + name: "the first three oridnals get to 3 nodes, the ordinal 3 should scheduled to 1,2,3", + ordinal: 3, + podFn: newHAPDPod, + nodesFn: threeNodes, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}, "kube-node-1": []int32{2}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).To(Equal(3)) + g.Expect(getSortedNodeNames(nodes)).To(Equal([]string{"kube-node-1", "kube-node-2", "kube-node-3"})) + }, + }, + { + name: "the first four oridnals get to 3 nodes, the ordinal 4 should scheduled to 2,3", + ordinal: 4, + podFn: newHAPDPod, + nodesFn: threeNodes, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}, "kube-node-1": []int32{2, 3}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).To(Equal(2)) + g.Expect(getSortedNodeNames(nodes)).To(Equal([]string{"kube-node-2", "kube-node-3"})) + }, + }, + { + name: "the first five oridnals get to 3 nodes, the ordinal 5 should scheduled to 2", + ordinal: 5, + podFn: newHAPDPod, + nodesFn: threeNodes, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).To(Equal(1)) + g.Expect(getSortedNodeNames(nodes)).To(Equal([]string{"kube-node-2"})) + }, + }, + { + name: "the first six oridnals get to 3 nodes, the ordinal 6 should scheduled to 1,2,3", + ordinal: 6, + podFn: newHAPDPod, + nodesFn: threeNodes, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0, 5}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).To(Equal(3)) + g.Expect(getSortedNodeNames(nodes)).To(Equal([]string{"kube-node-1", "kube-node-2", "kube-node-3"})) + }, + }, + } + + for i := range tests { + testFn(&tests[i], t) + } +} + +func newHAPDPod(clusterName string, ordinal int32) *apiv1.Pod { + return &apiv1.Pod{ + TypeMeta: metav1.TypeMeta{Kind: "Pod", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", controller.PDMemberName(clusterName), ordinal), + Namespace: corev1.NamespaceDefault, + Labels: label.New().PD().Labels(), + }, + } +} + +func threeNodes() []apiv1.Node { + return []apiv1.Node{ + { + TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "kube-node-1"}, + }, + { + TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "kube-node-2"}, + }, + { + TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "kube-node-3"}, + }, + } +} + +func podListFn(nodePodMap map[string][]int32) func(string, string, string) (*apiv1.PodList, error) { + return func(ns, clusterName, component string) (*apiv1.PodList, error) { + podList := &apiv1.PodList{ + TypeMeta: metav1.TypeMeta{Kind: "PodList", APIVersion: "v1"}, + Items: []apiv1.Pod{}, + } + for nodeName, podsOrdinalArr := range nodePodMap { + for _, podOrdinal := range podsOrdinalArr { + podList.Items = append(podList.Items, apiv1.Pod{ + TypeMeta: metav1.TypeMeta{Kind: "Pod", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", controller.PDMemberName(clusterName), podOrdinal), + Namespace: corev1.NamespaceDefault, + Labels: label.New().PD().Labels(), + }, + Spec: apiv1.PodSpec{ + NodeName: nodeName, + }, + }) + } + } + return podList, nil + } +} + +func podNameWrongListFn() func(string, string, string) (*apiv1.PodList, error) { + return func(ns, clusterName, component string) (*apiv1.PodList, error) { + podList := &apiv1.PodList{ + TypeMeta: metav1.TypeMeta{Kind: "PodList", APIVersion: "v1"}, + Items: []apiv1.Pod{ + { + TypeMeta: metav1.TypeMeta{Kind: "Pod", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "xxx", + Namespace: corev1.NamespaceDefault, + Labels: label.New().PD().Labels(), + }, + Spec: apiv1.PodSpec{ + NodeName: "kube-node-1", + }, + }, + }, + } + return podList, nil + } +} + +func podListErr() func(string, string, string) (*apiv1.PodList, error) { + return func(ns, clusterName, component string) (*apiv1.PodList, error) { + return nil, errors.New("pod list error") + } +} + +func getSortedNodeNames(nodes []apiv1.Node) []string { + arr := make([]string, 0) + for _, node := range nodes { + arr = append(arr, node.GetName()) + } + sort.Strings(arr) + return arr +} diff --git a/pkg/scheduler/predicates/predicate.go b/pkg/scheduler/predicates/predicate.go index 8d22016e538..a596a0b0c53 100644 --- a/pkg/scheduler/predicates/predicate.go +++ b/pkg/scheduler/predicates/predicate.go @@ -22,9 +22,19 @@ type Predicate interface { // Name return the predicate name Name() string - // Filter function receives a *volume.TiDBVolume, an *apiv1.Pod and an *apiv1.PersistentVolumeClaim, - // should remove or add a Priority to every volumes, - // and return whether it is fusing after this predicate, and an error. - // Implementations must treat the *apiv1.Pod and *apiv1.PersistentVolumeClaim parameter as read-only and not modify it. - Filter(*apiv1.Pod) (bool, error) + // Filter function receives a set of nodes and returns a set of candidate nodes. + Filter(string, *apiv1.Pod, []apiv1.Node) ([]apiv1.Node, error) +} + +func getNodeFromNames(nodes []apiv1.Node, nodeNames []string) []apiv1.Node { + var retNodes []apiv1.Node + for _, node := range nodes { + for _, nodeName := range nodeNames { + if node.GetName() == nodeName { + retNodes = append(retNodes, node) + break + } + } + } + return retNodes } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 03f8c0ade97..043ffd0d0e9 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -14,6 +14,12 @@ package scheduler import ( + "fmt" + + "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/scheduler/predicates" + apiv1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" schedulerapiv1 "k8s.io/kubernetes/pkg/scheduler/api/v1" ) @@ -33,21 +39,21 @@ type Scheduler interface { } type scheduler struct { - kubeCli kubernetes.Interface - // predicates []predicates.Predicate + kubeCli kubernetes.Interface + predicates []predicates.Predicate } // NewScheduler returns a Scheduler -func NewScheduler(kubeCli kubernetes.Interface) Scheduler { +func NewScheduler(kubeCli kubernetes.Interface, pdReplicas, tikvReplicas int32) Scheduler { return &scheduler{ kubeCli: kubeCli, - // predicates: []predicates.Predicate{ - // highavailability.NewHighAvailability(10, kubeCli), - //}, + predicates: []predicates.Predicate{ + predicates.NewHA(kubeCli, pdReplicas, tikvReplicas), + }, } } -// Filter select a node from *schedulerapiv1.ExtenderArgs.Nodes when this is a pd or tikv pod +// Filter selects a set of nodes from *schedulerapiv1.ExtenderArgs.Nodes when this is a pd or tikv pod // else return the original nodes. func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.ExtenderFilterResult, error) { pod := &args.Pod @@ -55,7 +61,32 @@ func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.E podName := pod.GetName() kubeNodes := args.Nodes.Items - return nil, nil + var clusterName string + var exist bool + if clusterName, exist = pod.Labels[label.InstanceLabelKey]; !exist { + return nil, fmt.Errorf("can't find clusterName in pod labels: %s/%s", ns, podName) + } + if component := pod.Labels[label.ComponentLabelKey]; component != label.PDLabelVal && component != label.TiKVLabelVal { + return &schedulerapiv1.ExtenderFilterResult{ + Nodes: args.Nodes, + NodeNames: args.NodeNames, + }, nil + } + + glog.Infof("scheduling pod: %s/%s", ns, podName) + var err error + for _, predicate := range s.predicates { + glog.Infof("entering predicate: %s, nodes: %v", predicate.Name(), getNodeNames(kubeNodes)) + kubeNodes, err = predicate.Filter(clusterName, pod, kubeNodes) + if err != nil { + return nil, err + } + glog.Infof("leaving predicate: %s, nodes: %v", predicate.Name(), getNodeNames(kubeNodes)) + } + + return &schedulerapiv1.ExtenderFilterResult{ + Nodes: &apiv1.NodeList{Items: kubeNodes}, + }, nil } // We didn't pass `prioritizeVerb` to kubernetes scheduler extender's config file, this method will not be called. @@ -74,3 +105,11 @@ func (s *scheduler) Priority(args *schedulerapiv1.ExtenderArgs) (schedulerapiv1. } var _ Scheduler = &scheduler{} + +func getNodeNames(nodes []apiv1.Node) []string { + nodeNames := make([]string, 0) + for _, node := range nodes { + nodeNames = append(nodeNames, node.GetName()) + } + return nodeNames +} diff --git a/pkg/scheduler/server/mux.go b/pkg/scheduler/server/mux.go index 3ae71157bc6..d74d6719ddc 100644 --- a/pkg/scheduler/server/mux.go +++ b/pkg/scheduler/server/mux.go @@ -18,10 +18,9 @@ import ( "net/http" "sync" - "github.com/pingcap/tidb-operator/pkg/scheduler" - restful "github.com/emicklei/go-restful" "github.com/golang/glog" + "github.com/pingcap/tidb-operator/pkg/scheduler" "k8s.io/client-go/kubernetes" schedulerapiv1 "k8s.io/kubernetes/pkg/scheduler/api/v1" ) @@ -33,13 +32,12 @@ var ( type server struct { scheduler scheduler.Scheduler - // All scheduling should be serialized - lock sync.Mutex + lock sync.Mutex } -// StartServer start a kubernetes scheduler extender http apiserver -func StartServer(kubeCli kubernetes.Interface, port int) { - s := scheduler.NewScheduler(kubeCli) +// StartServer starts a kubernetes scheduler extender http apiserver +func StartServer(kubeCli kubernetes.Interface, port int, pdReplicas, tikvReplicas int32) { + s := scheduler.NewScheduler(kubeCli, pdReplicas, tikvReplicas) svr := &server{scheduler: s} ws := new(restful.WebService) diff --git a/pkg/util/util.go b/pkg/util/util.go index af8ab47259d..1142ffac9b2 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -15,6 +15,7 @@ package util import ( "sort" + "strconv" "strings" "github.com/golang/glog" @@ -206,3 +207,12 @@ func ResourceRequirement(spec v1alpha1.ContainerSpec, defaultRequests ...corev1. } return rr } + +func GetOrdinalFromPodName(podName string) (int32, error) { + ordinalStr := podName[strings.LastIndex(podName, "-")+1:] + ordinalInt, err := strconv.Atoi(ordinalStr) + if err != nil { + return int32(0), err + } + return int32(ordinalInt), nil +} From b5daa753f061e41df33a1a371127d0b7de100c44 Mon Sep 17 00:00:00 2001 From: weekface Date: Thu, 25 Oct 2018 17:57:37 +0800 Subject: [PATCH 03/12] add e2e test specs --- tests/e2e/create.go | 12 ++++++++++++ tests/e2e/scale.go | 16 ++++++++++++++++ tests/e2e/upgrade.go | 19 +++++++++++++++++-- tests/e2e/utils.go | 23 +++++++++++++++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/tests/e2e/create.go b/tests/e2e/create.go index 0c000723206..087c64a43cb 100644 --- a/tests/e2e/create.go +++ b/tests/e2e/create.go @@ -35,6 +35,18 @@ func testCreate() { err := wait.Poll(5*time.Second, 5*time.Minute, allMembersRunning) Expect(err).NotTo(HaveOccurred()) + By("And scheduling policy is correct") + nodeMap, err := getNodeMap(label.PDLabelVal) + Expect(err).NotTo(HaveOccurred()) + for _, podNamesArr := range nodeMap { + Expect(len(podNamesArr)).To(Equal(1)) + } + nodeMap, err = getNodeMap(label.TiKVLabelVal) + Expect(err).NotTo(HaveOccurred()) + for _, podNamesArr := range nodeMap { + Expect(len(podNamesArr)).To(Equal(1)) + } + By("When create a table and add some data to this table") err = wait.Poll(5*time.Second, 5*time.Minute, addDataToCluster) Expect(err).NotTo(HaveOccurred()) diff --git a/tests/e2e/scale.go b/tests/e2e/scale.go index 0bf337230b6..b6e44740a4a 100644 --- a/tests/e2e/scale.go +++ b/tests/e2e/scale.go @@ -50,6 +50,22 @@ func testScale() { err = wait.Poll(5*time.Second, 5*time.Minute, scaledCorrectly) Expect(err).NotTo(HaveOccurred()) + By("And scheduling policy is correct") + nodeMap, err := getNodeMap(label.PDLabelVal) + Expect(err).NotTo(HaveOccurred()) + for nodeName, podNamesArr := range nodeMap { + if len(podNamesArr) > 2 { + Fail(fmt.Sprintf("node: %s has %d pods", nodeName, len(podNamesArr))) + } + } + nodeMap, err = getNodeMap(label.TiKVLabelVal) + Expect(err).NotTo(HaveOccurred()) + for nodeName, podNamesArr := range nodeMap { + if len(podNamesArr) > 2 { + Fail(fmt.Sprintf("node: %s has %d pods", nodeName, len(podNamesArr))) + } + } + By("And the data is correct") err = wait.Poll(5*time.Second, 5*time.Minute, dataIsCorrect) Expect(err).NotTo(HaveOccurred()) diff --git a/tests/e2e/upgrade.go b/tests/e2e/upgrade.go index c6520d38540..4cf6d6273f7 100644 --- a/tests/e2e/upgrade.go +++ b/tests/e2e/upgrade.go @@ -21,18 +21,24 @@ import ( . "github.com/onsi/gomega" // revive:disable:dot-imports "github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/label" apps "k8s.io/api/apps/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" ) const ( - upgradeVersion = "v2.1.0-rc.3" + upgradeVersion = "v2.0.8" ) func testUpgrade() { + pdNodeMap, err := getNodeMap(label.PDLabelVal) + Expect(err).NotTo(HaveOccurred()) + tikvNodeMap, err := getNodeMap(label.TiKVLabelVal) + Expect(err).NotTo(HaveOccurred()) + By("When upgrade TiDB cluster to newer version") - err := wait.Poll(5*time.Second, 5*time.Minute, upgrade) + err = wait.Poll(5*time.Second, 5*time.Minute, upgrade) Expect(err).NotTo(HaveOccurred()) By("Then members should be upgrade in order: pd ==> tikv ==> tidb") @@ -43,6 +49,15 @@ func testUpgrade() { err = wait.Poll(5*time.Second, 5*time.Minute, allMembersRunning) Expect(err).NotTo(HaveOccurred()) + By("And scheduling policy is correct") + pdNodeMap1, err := getNodeMap(label.PDLabelVal) + Expect(err).NotTo(HaveOccurred()) + tikvNodeMap1, err := getNodeMap(label.TiKVLabelVal) + Expect(err).NotTo(HaveOccurred()) + + Expect(pdNodeMap).To(Equal(pdNodeMap1)) + Expect(tikvNodeMap).To(Equal(tikvNodeMap1)) + By("And the data is correct") err = wait.Poll(5*time.Second, 5*time.Minute, dataIsCorrect) Expect(err).NotTo(HaveOccurred()) diff --git a/tests/e2e/utils.go b/tests/e2e/utils.go index 924022d30d5..cfffec1b36a 100644 --- a/tests/e2e/utils.go +++ b/tests/e2e/utils.go @@ -16,6 +16,7 @@ package e2e import ( "fmt" "os/exec" + "sort" "strings" "time" @@ -170,3 +171,25 @@ func logf(format string, args ...interface{}) { func isNotFound(err error) bool { return strings.Contains(err.Error(), "not found") } + +func getNodeMap(component string) (map[string][]string, error) { + nodeMap := make(map[string][]string) + selector := label.New().Cluster(clusterName).Component(component).Labels() + podList, err := kubeCli.CoreV1().Pods(ns).List(metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(selector).String(), + }) + if err != nil { + return nil, err + } + + for _, pod := range podList.Items { + nodeName := pod.Spec.NodeName + if len(nodeMap[nodeName]) == 0 { + nodeMap[nodeName] = make([]string, 0) + } + nodeMap[nodeName] = append(nodeMap[nodeName], pod.GetName()) + sort.Strings(nodeMap[nodeName]) + } + + return nodeMap, nil +} From dfbb8d7bfa0be165580047358cc2768f91b0311d Mon Sep 17 00:00:00 2001 From: weekface Date: Fri, 26 Oct 2018 22:49:26 +0800 Subject: [PATCH 04/12] * add more unit tests * address comment --- pkg/scheduler/predicates/ha.go | 3 + pkg/scheduler/predicates/ha_test.go | 147 +++++++++++++++++---- pkg/scheduler/predicates/predicate_test.go | 127 ++++++++++++++++++ pkg/scheduler/predicates/test_helper.go | 49 +++++++ 4 files changed, 297 insertions(+), 29 deletions(-) create mode 100644 pkg/scheduler/predicates/predicate_test.go create mode 100644 pkg/scheduler/predicates/test_helper.go diff --git a/pkg/scheduler/predicates/ha.go b/pkg/scheduler/predicates/ha.go index 4872e6ca778..779de11a496 100644 --- a/pkg/scheduler/predicates/ha.go +++ b/pkg/scheduler/predicates/ha.go @@ -83,6 +83,9 @@ func (h *ha) Filter(clusterName string, pod *apiv1.Pod, nodes []apiv1.Node) ([]a if nodeName == "" { continue } + if nodeMap[nodeName] == nil { + continue + } nodeMap[nodeName] = append(nodeMap[nodeName], podName1) } diff --git a/pkg/scheduler/predicates/ha_test.go b/pkg/scheduler/predicates/ha_test.go index b23618a68a7..6c2c4c1456f 100644 --- a/pkg/scheduler/predicates/ha_test.go +++ b/pkg/scheduler/predicates/ha_test.go @@ -28,6 +28,19 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func TestMapNil(t *testing.T) { + g := NewGomegaWithT(t) + + m := make(map[string][]string) + arr := []string{"a", "b"} + for _, item := range arr { + m[item] = make([]string, 0) + } + + g.Expect(m["c"] == nil).To(Equal(true)) + g.Expect(m["a"] == nil).To(Equal(false)) +} + func TestHAFilter(t *testing.T) { g := NewGomegaWithT(t) type testcase struct { @@ -61,7 +74,7 @@ func TestHAFilter(t *testing.T) { pod.Labels = nil return pod }, - nodesFn: threeNodes, + nodesFn: fakeThreeNodes, podListFn: podListFn(map[string][]int32{}), pdReplicas: 3, tikvReplicas: 3, @@ -73,7 +86,7 @@ func TestHAFilter(t *testing.T) { name: "pod list return error", ordinal: 0, podFn: newHAPDPod, - nodesFn: threeNodes, + nodesFn: fakeThreeNodes, podListFn: podListErr(), pdReplicas: 3, tikvReplicas: 3, @@ -89,7 +102,7 @@ func TestHAFilter(t *testing.T) { pod.Name = "xxxx" return pod }, - nodesFn: threeNodes, + nodesFn: fakeThreeNodes, podListFn: podListFn(map[string][]int32{}), pdReplicas: 3, tikvReplicas: 3, @@ -101,7 +114,7 @@ func TestHAFilter(t *testing.T) { name: "one pod, podName is wrong", ordinal: 0, podFn: newHAPDPod, - nodesFn: threeNodes, + nodesFn: fakeThreeNodes, podListFn: podNameWrongListFn(), pdReplicas: 3, tikvReplicas: 3, @@ -113,7 +126,7 @@ func TestHAFilter(t *testing.T) { name: "the lower oridnal is not scheduled", ordinal: 1, podFn: newHAPDPod, - nodesFn: threeNodes, + nodesFn: fakeThreeNodes, podListFn: podListFn(map[string][]int32{"": []int32{0}}), pdReplicas: 3, tikvReplicas: 3, @@ -125,7 +138,7 @@ func TestHAFilter(t *testing.T) { name: "no scheduled pods, three nodes, ordinal 0 should be scheduled to all nodes", ordinal: 0, podFn: newHAPDPod, - nodesFn: threeNodes, + nodesFn: fakeThreeNodes, podListFn: podListFn(map[string][]int32{}), pdReplicas: 3, tikvReplicas: 3, @@ -139,7 +152,7 @@ func TestHAFilter(t *testing.T) { name: "ordinal 0 is scheduled to kube-node-1, ordinal 1 should be scheduled to kube-node-2 or kube-node-3", ordinal: 1, podFn: newHAPDPod, - nodesFn: threeNodes, + nodesFn: fakeThreeNodes, podListFn: podListFn(map[string][]int32{"kube-node-1": []int32{0}}), pdReplicas: 3, tikvReplicas: 3, @@ -149,11 +162,37 @@ func TestHAFilter(t *testing.T) { g.Expect(getSortedNodeNames(nodes)).To(Equal([]string{"kube-node-2", "kube-node-3"})) }, }, + { + name: "ordinal 0 is scheduled to kube-node-3, get node-3, ordinal 1 should be scheduled to none", + ordinal: 1, + podFn: newHAPDPod, + nodesFn: fakeOneNode, + podListFn: podListFn(map[string][]int32{"kube-node-3": []int32{0}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(strings.Contains(err.Error(), "the first 3 pods can't scheduled to the same node")).To(Equal(true)) + }, + }, + { + name: "ordinal 0 is scheduled to kube-node-3, get 0 node, ordinal 1 should be scheduled to none", + ordinal: 1, + podFn: newHAPDPod, + nodesFn: fakeZeroNode, + podListFn: podListFn(map[string][]int32{"kube-node-3": []int32{0}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(strings.Contains(err.Error(), "no suitable node for pod: default/demo-pd-1")).To(Equal(true)) + }, + }, { name: "ordinal 0 is scheduled to kube-node-2, ordinal 1 is kube-node-3, ordinal 2 should be scheduled to kube-node-1", ordinal: 2, podFn: newHAPDPod, - nodesFn: threeNodes, + nodesFn: fakeThreeNodes, podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}}), pdReplicas: 3, tikvReplicas: 3, @@ -163,11 +202,37 @@ func TestHAFilter(t *testing.T) { g.Expect(getSortedNodeNames(nodes)).To(Equal([]string{"kube-node-1"})) }, }, + { + name: "ordinal 0 is scheduled to kube-node-1, ordinal 1 is kube-node-3, ordinal 2 should be scheduled to kube-node-1", + ordinal: 2, + podFn: newHAPDPod, + nodesFn: fakeTwoNodes, + podListFn: podListFn(map[string][]int32{"kube-node-1": []int32{0}, "kube-node-3": []int32{1}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(strings.Contains(err.Error(), "the first 3 pods can't scheduled to the same node")).To(Equal(true)) + }, + }, + { + name: "ordinal 0 is scheduled to kube-node-1, ordinal 1 is kube-node-3, get 0 node, ordinal 2 should be scheduled to none", + ordinal: 2, + podFn: newHAPDPod, + nodesFn: fakeZeroNode, + podListFn: podListFn(map[string][]int32{"kube-node-1": []int32{0}, "kube-node-3": []int32{1}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(strings.Contains(err.Error(), "no suitable node for pod: default/demo-pd-2")).To(Equal(true)) + }, + }, { name: "the first three oridnals get to 3 nodes, the ordinal 3 should scheduled to 1,2,3", ordinal: 3, podFn: newHAPDPod, - nodesFn: threeNodes, + nodesFn: fakeThreeNodes, podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}, "kube-node-1": []int32{2}}), pdReplicas: 3, tikvReplicas: 3, @@ -181,7 +246,7 @@ func TestHAFilter(t *testing.T) { name: "the first four oridnals get to 3 nodes, the ordinal 4 should scheduled to 2,3", ordinal: 4, podFn: newHAPDPod, - nodesFn: threeNodes, + nodesFn: fakeThreeNodes, podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}, "kube-node-1": []int32{2, 3}}), pdReplicas: 3, tikvReplicas: 3, @@ -195,7 +260,7 @@ func TestHAFilter(t *testing.T) { name: "the first five oridnals get to 3 nodes, the ordinal 5 should scheduled to 2", ordinal: 5, podFn: newHAPDPod, - nodesFn: threeNodes, + nodesFn: fakeThreeNodes, podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), pdReplicas: 3, tikvReplicas: 3, @@ -205,11 +270,52 @@ func TestHAFilter(t *testing.T) { g.Expect(getSortedNodeNames(nodes)).To(Equal([]string{"kube-node-2"})) }, }, + { + name: "the first five oridnals get to 3 nodes, got 2 nodes(no node-2), the ordinal 5 should scheduled to 1,3", + ordinal: 5, + podFn: newHAPDPod, + nodesFn: fakeTwoNodes, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).To(Equal(2)) + g.Expect(getSortedNodeNames(nodes)).To(Equal([]string{"kube-node-1", "kube-node-3"})) + }, + }, + { + name: "the first five oridnals get to 3 nodes, got 1 nodes(no node-2 node-1), the ordinal 5 should scheduled to 3", + ordinal: 5, + podFn: newHAPDPod, + nodesFn: fakeOneNode, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).To(Equal(1)) + g.Expect(getSortedNodeNames(nodes)).To(Equal([]string{"kube-node-3"})) + }, + }, + { + name: "the first five oridnals get to 3 nodes, got 0 nodes, the ordinal 5 should scheduled to none", + ordinal: 5, + podFn: newHAPDPod, + nodesFn: fakeZeroNode, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), + pdReplicas: 3, + tikvReplicas: 3, + expectFn: func(nodes []apiv1.Node, err error) { + g.Expect(err).To(HaveOccurred()) + g.Expect(strings.Contains(err.Error(), "no suitable node for pod: default/demo-pd-5")).To(Equal(true)) + }, + }, { name: "the first six oridnals get to 3 nodes, the ordinal 6 should scheduled to 1,2,3", ordinal: 6, podFn: newHAPDPod, - nodesFn: threeNodes, + nodesFn: fakeThreeNodes, podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0, 5}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), pdReplicas: 3, tikvReplicas: 3, @@ -237,23 +343,6 @@ func newHAPDPod(clusterName string, ordinal int32) *apiv1.Pod { } } -func threeNodes() []apiv1.Node { - return []apiv1.Node{ - { - TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "kube-node-1"}, - }, - { - TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "kube-node-2"}, - }, - { - TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: "v1"}, - ObjectMeta: metav1.ObjectMeta{Name: "kube-node-3"}, - }, - } -} - func podListFn(nodePodMap map[string][]int32) func(string, string, string) (*apiv1.PodList, error) { return func(ns, clusterName, component string) (*apiv1.PodList, error) { podList := &apiv1.PodList{ diff --git a/pkg/scheduler/predicates/predicate_test.go b/pkg/scheduler/predicates/predicate_test.go new file mode 100644 index 00000000000..2332a72108e --- /dev/null +++ b/pkg/scheduler/predicates/predicate_test.go @@ -0,0 +1,127 @@ +// Copyright 2018 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package predicates + +import ( + "testing" + + . "github.com/onsi/gomega" + apiv1 "k8s.io/api/core/v1" +) + +func TestGetNodeFromNames(t *testing.T) { + g := NewGomegaWithT(t) + + type testcase struct { + nodes []apiv1.Node + nodeNames []string + expected []string + } + + testFn := func(i int, test *testcase, t *testing.T) { + t.Log(i) + arr := getNodeFromNames(test.nodes, test.nodeNames) + g.Expect(len(arr)).To(Equal(len(test.expected))) + for idx, node := range arr { + g.Expect(node.GetName()).To(Equal(test.expected[idx])) + } + } + + tests := []testcase{ + { + nodes: fakeThreeNodes(), + nodeNames: []string{}, + expected: []string{}, + }, + { + nodes: fakeThreeNodes(), + nodeNames: []string{"kube-node-1"}, + expected: []string{"kube-node-1"}, + }, + { + nodes: fakeThreeNodes(), + nodeNames: []string{"kube-node-1", "kube-node-2"}, + expected: []string{"kube-node-1", "kube-node-2"}, + }, + { + nodes: fakeThreeNodes(), + nodeNames: []string{"kube-node-1", "kube-node-2", "kube-node-3"}, + expected: []string{"kube-node-1", "kube-node-2", "kube-node-3"}, + }, + { + nodes: fakeTwoNodes(), + nodeNames: []string{"kube-node-1", "kube-node-2", "kube-node-3"}, + expected: []string{"kube-node-1", "kube-node-3"}, + }, + { + nodes: fakeTwoNodes(), + nodeNames: []string{"kube-node-1", "kube-node-3"}, + expected: []string{"kube-node-1", "kube-node-3"}, + }, + { + nodes: fakeTwoNodes(), + nodeNames: []string{"kube-node-1"}, + expected: []string{"kube-node-1"}, + }, + { + nodes: fakeTwoNodes(), + nodeNames: []string{"kube-node-3"}, + expected: []string{"kube-node-3"}, + }, + { + nodes: fakeTwoNodes(), + nodeNames: []string{"kube-node-2"}, + expected: []string{}, + }, + { + nodes: fakeOneNode(), + nodeNames: []string{"kube-node-1", "kube-node-2", "kube-node-3"}, + expected: []string{"kube-node-3"}, + }, + { + nodes: fakeOneNode(), + nodeNames: []string{"kube-node-2", "kube-node-3"}, + expected: []string{"kube-node-3"}, + }, + { + nodes: fakeOneNode(), + nodeNames: []string{"kube-node-1", "kube-node-3"}, + expected: []string{"kube-node-3"}, + }, + { + nodes: fakeOneNode(), + nodeNames: []string{"kube-node-1", "kube-node-2"}, + expected: []string{}, + }, + { + nodes: fakeZeroNode(), + nodeNames: []string{"kube-node-1", "kube-node-2", "kube-node-3"}, + expected: []string{}, + }, + { + nodes: fakeZeroNode(), + nodeNames: []string{"kube-node-2", "kube-node-3"}, + expected: []string{}, + }, + { + nodes: fakeZeroNode(), + nodeNames: []string{"kube-node-3"}, + expected: []string{}, + }, + } + + for i := range tests { + testFn(i, &tests[i], t) + } +} diff --git a/pkg/scheduler/predicates/test_helper.go b/pkg/scheduler/predicates/test_helper.go new file mode 100644 index 00000000000..ab349ab40bd --- /dev/null +++ b/pkg/scheduler/predicates/test_helper.go @@ -0,0 +1,49 @@ +package predicates + +import ( + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func fakeThreeNodes() []apiv1.Node { + return []apiv1.Node{ + { + TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "kube-node-1"}, + }, + { + TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "kube-node-2"}, + }, + { + TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "kube-node-3"}, + }, + } +} + +func fakeTwoNodes() []apiv1.Node { + return []apiv1.Node{ + { + TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "kube-node-1"}, + }, + { + TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "kube-node-3"}, + }, + } +} + +func fakeOneNode() []apiv1.Node { + return []apiv1.Node{ + { + TypeMeta: metav1.TypeMeta{Kind: "Node", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "kube-node-3"}, + }, + } +} + +func fakeZeroNode() []apiv1.Node { + return []apiv1.Node{} +} From 728d347d121cf8927131d2404eb2d521903d015d Mon Sep 17 00:00:00 2001 From: weekface Date: Tue, 30 Oct 2018 11:50:13 +0800 Subject: [PATCH 05/12] address comment --- .../templates/scheduler-deployment.yaml | 18 +- charts/tidb-operator/values.yaml | 7 +- cmd/scheduler/main.go | 6 +- pkg/scheduler/predicates/ha.go | 29 +-- pkg/scheduler/predicates/ha_test.go | 236 ++++++++---------- pkg/scheduler/scheduler.go | 19 +- pkg/scheduler/server/mux.go | 4 +- 7 files changed, 124 insertions(+), 195 deletions(-) diff --git a/charts/tidb-operator/templates/scheduler-deployment.yaml b/charts/tidb-operator/templates/scheduler-deployment.yaml index 07b15c336e4..59d3caa375d 100644 --- a/charts/tidb-operator/templates/scheduler-deployment.yaml +++ b/charts/tidb-operator/templates/scheduler-deployment.yaml @@ -1,4 +1,3 @@ -{{- $defaultHyperkubeImage := "quay.io/coreos/hyperkube:v1.10.4_coreos.0" -}} apiVersion: extensions/v1beta1 kind: Deployment metadata: @@ -35,30 +34,17 @@ spec: - /usr/local/bin/tidb-scheduler - -v={{ .Values.scheduler.logLevel }} - -port=10262 - - -pd-replicas={{ .Values.scheduler.pdReplicas | default 3 }} - - -tikv-replicas={{ .Values.scheduler.tikvReplicas | default 3 }} - name: kube-scheduler - {{- if .Values.scheduler.kubeSchedulerImage }} image: {{ .Values.scheduler.kubeSchedulerImage }} - {{- else if .Values.scheduler.hyperkubeImage }} - image: {{ .Values.scheduler.hyperkubeImage }} - {{- else }} - image: {{ $defaultHyperkubeImage }} - {{- end }} resources: {{ toYaml .Values.scheduler.resources | indent 12 }} command: - {{- if .Values.scheduler.kubeSchedulerImage }} - kube-scheduler - {{- else }} - - /hyperkube - - scheduler - {{- end }} - --port=10261 - --leader-elect=true - - --lock-object-name=tidb-scheduler + - --lock-object-name={{ .Values.scheduler.schedulerName }} - --lock-object-namespace={{ .Release.Namespace }} - --scheduler-name={{ .Values.scheduler.schedulerName }} - --v={{ .Values.scheduler.logLevel }} - - --policy-configmap=tidb-scheduler-policy + - --policy-configmap={{ .Values.scheduler.schedulerName }}-policy - --policy-configmap-namespace={{ .Release.Namespace }} diff --git a/charts/tidb-operator/values.yaml b/charts/tidb-operator/values.yaml index a5c2c5c645b..dbf46d1b947 100644 --- a/charts/tidb-operator/values.yaml +++ b/charts/tidb-operator/values.yaml @@ -50,9 +50,4 @@ scheduler: requests: cpu: 80m memory: 50Mi - # pd replicas - pdReplicas: 3 - # tikv replicas - tikvReplicas: 3 - hyperkubeImage: quay.io/coreos/hyperkube:v1.10.4_coreos.0 - # kubeSchedulerImage: + kubeSchedulerImage: quay.io/coreos/hyperkube:v1.10.4_coreos.0 diff --git a/cmd/scheduler/main.go b/cmd/scheduler/main.go index 01a6687a015..f20d4cf4b08 100644 --- a/cmd/scheduler/main.go +++ b/cmd/scheduler/main.go @@ -32,16 +32,12 @@ import ( var ( printVersion bool port int - pdReplicas int - tikvReplicas int ) func init() { flag.BoolVar(&printVersion, "V", false, "Show version and quit") flag.BoolVar(&printVersion, "version", false, "Show version and quit") flag.IntVar(&port, "port", 10262, "The port that the tidb scheduler's http service runs on (default 10262)") - flag.IntVar(&pdReplicas, "pd-replicas", 3, "The pd replicas (default 3)") - flag.IntVar(&tikvReplicas, "tikv-replicas", 3, "The tikv replicas (default 3)") flag.Parse() } @@ -65,7 +61,7 @@ func main() { } go wait.Forever(func() { - server.StartServer(kubeCli, port, int32(pdReplicas), int32(tikvReplicas)) + server.StartServer(kubeCli, port) }, 5*time.Second) glog.Fatal(http.ListenAndServe(":6060", nil)) } diff --git a/pkg/scheduler/predicates/ha.go b/pkg/scheduler/predicates/ha.go index 779de11a496..01b8dd1acc2 100644 --- a/pkg/scheduler/predicates/ha.go +++ b/pkg/scheduler/predicates/ha.go @@ -24,19 +24,19 @@ import ( "k8s.io/client-go/kubernetes" ) +const ( + replicas int32 = 3 +) + type ha struct { - kubeCli kubernetes.Interface - podListFn func(ns, clusterName, component string) (*apiv1.PodList, error) - pdReplicas int32 - tikvReplicas int32 + kubeCli kubernetes.Interface + podListFn func(ns, clusterName, component string) (*apiv1.PodList, error) } // NewHA returns a Predicate -func NewHA(kubeCli kubernetes.Interface, pdReplicas, tikvReplicas int32) Predicate { +func NewHA(kubeCli kubernetes.Interface) Predicate { h := &ha{ - kubeCli: kubeCli, - pdReplicas: pdReplicas, - tikvReplicas: tikvReplicas, + kubeCli: kubeCli, } h.podListFn = h.realPodListFn return h @@ -53,7 +53,7 @@ func (h *ha) Filter(clusterName string, pod *apiv1.Pod, nodes []apiv1.Node) ([]a var component string var exist bool if component, exist = pod.Labels[label.ComponentLabelKey]; !exist { - return nil, fmt.Errorf("can't find component in pod labels: %s/%s", ns, podName) + return nodes, fmt.Errorf("can't find component in pod labels: %s/%s", ns, podName) } podList, err := h.podListFn(ns, clusterName, component) if err != nil { @@ -111,17 +111,8 @@ func (h *ha) Filter(clusterName string, pod *apiv1.Pod, nodes []apiv1.Node) ([]a if len(minNodeNames) == 0 { return nil, fmt.Errorf("no suitable node for pod: %s/%s", ns, podName) } - - var replicas int32 - switch component { - case label.PDLabelVal: - replicas = h.pdReplicas - case label.TiKVLabelVal: - replicas = h.tikvReplicas - } - if ordinal < replicas && min != 0 { - return nil, fmt.Errorf("the first %d pods can't scheduled to the same node", replicas) + return nil, fmt.Errorf("the first %d pods can't be scheduled to the same node", replicas) } return getNodeFromNames(nodes, minNodeNames), nil diff --git a/pkg/scheduler/predicates/ha_test.go b/pkg/scheduler/predicates/ha_test.go index 6c2c4c1456f..9ba056cbda1 100644 --- a/pkg/scheduler/predicates/ha_test.go +++ b/pkg/scheduler/predicates/ha_test.go @@ -44,14 +44,12 @@ func TestMapNil(t *testing.T) { func TestHAFilter(t *testing.T) { g := NewGomegaWithT(t) type testcase struct { - name string - ordinal int32 - podFn func(string, int32) *apiv1.Pod - nodesFn func() []apiv1.Node - podListFn func(string, string, string) (*apiv1.PodList, error) - pdReplicas int32 - tikvReplicas int32 - expectFn func([]apiv1.Node, error) + name string + ordinal int32 + podFn func(string, int32) *apiv1.Pod + nodesFn func() []apiv1.Node + podListFn func(string, string, string) (*apiv1.PodList, error) + expectFn func([]apiv1.Node, error) } testFn := func(test *testcase, t *testing.T) { @@ -61,7 +59,7 @@ func TestHAFilter(t *testing.T) { pod := test.podFn(clusterName, test.ordinal) nodes := test.nodesFn() - ha := ha{podListFn: test.podListFn, pdReplicas: test.pdReplicas, tikvReplicas: test.tikvReplicas} + ha := ha{podListFn: test.podListFn} test.expectFn(ha.Filter(clusterName, pod, nodes)) } @@ -74,22 +72,18 @@ func TestHAFilter(t *testing.T) { pod.Labels = nil return pod }, - nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{}), - pdReplicas: 3, - tikvReplicas: 3, + nodesFn: fakeThreeNodes, + podListFn: podListFn(map[string][]int32{}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(strings.Contains(err.Error(), "can't find component in pod labels")).To(Equal(true)) }, }, { - name: "pod list return error", - ordinal: 0, - podFn: newHAPDPod, - nodesFn: fakeThreeNodes, - podListFn: podListErr(), - pdReplicas: 3, - tikvReplicas: 3, + name: "pod list return error", + ordinal: 0, + podFn: newHAPDPod, + nodesFn: fakeThreeNodes, + podListFn: podListErr(), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(strings.Contains(err.Error(), "pod list error")).To(Equal(true)) }, @@ -102,46 +96,38 @@ func TestHAFilter(t *testing.T) { pod.Name = "xxxx" return pod }, - nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{}), - pdReplicas: 3, - tikvReplicas: 3, + nodesFn: fakeThreeNodes, + podListFn: podListFn(map[string][]int32{}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(strings.Contains(err.Error(), "strconv.Atoi: parsing")).To(Equal(true)) }, }, { - name: "one pod, podName is wrong", - ordinal: 0, - podFn: newHAPDPod, - nodesFn: fakeThreeNodes, - podListFn: podNameWrongListFn(), - pdReplicas: 3, - tikvReplicas: 3, + name: "one pod, podName is wrong", + ordinal: 0, + podFn: newHAPDPod, + nodesFn: fakeThreeNodes, + podListFn: podNameWrongListFn(), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(strings.Contains(err.Error(), "strconv.Atoi: parsing")).To(Equal(true)) }, }, { - name: "the lower oridnal is not scheduled", - ordinal: 1, - podFn: newHAPDPod, - nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"": []int32{0}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "the lower oridnal is not scheduled", + ordinal: 1, + podFn: newHAPDPod, + nodesFn: fakeThreeNodes, + podListFn: podListFn(map[string][]int32{"": []int32{0}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(strings.Contains(err.Error(), "waiting for pod: default/demo-pd-0")).To(Equal(true)) }, }, { - name: "no scheduled pods, three nodes, ordinal 0 should be scheduled to all nodes", - ordinal: 0, - podFn: newHAPDPod, - nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{}), - pdReplicas: 3, - tikvReplicas: 3, + name: "no scheduled pods, three nodes, ordinal 0 should be scheduled to all nodes", + ordinal: 0, + podFn: newHAPDPod, + nodesFn: fakeThreeNodes, + podListFn: podListFn(map[string][]int32{}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(3)) @@ -149,13 +135,11 @@ func TestHAFilter(t *testing.T) { }, }, { - name: "ordinal 0 is scheduled to kube-node-1, ordinal 1 should be scheduled to kube-node-2 or kube-node-3", - ordinal: 1, - podFn: newHAPDPod, - nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"kube-node-1": []int32{0}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "ordinal 0 is scheduled to kube-node-1, ordinal 1 should be scheduled to kube-node-2 or kube-node-3", + ordinal: 1, + podFn: newHAPDPod, + nodesFn: fakeThreeNodes, + podListFn: podListFn(map[string][]int32{"kube-node-1": []int32{0}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(2)) @@ -163,39 +147,33 @@ func TestHAFilter(t *testing.T) { }, }, { - name: "ordinal 0 is scheduled to kube-node-3, get node-3, ordinal 1 should be scheduled to none", - ordinal: 1, - podFn: newHAPDPod, - nodesFn: fakeOneNode, - podListFn: podListFn(map[string][]int32{"kube-node-3": []int32{0}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "ordinal 0 is scheduled to kube-node-3, get node-3, ordinal 1 should be scheduled to none", + ordinal: 1, + podFn: newHAPDPod, + nodesFn: fakeOneNode, + podListFn: podListFn(map[string][]int32{"kube-node-3": []int32{0}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).To(HaveOccurred()) - g.Expect(strings.Contains(err.Error(), "the first 3 pods can't scheduled to the same node")).To(Equal(true)) + g.Expect(strings.Contains(err.Error(), "the first 3 pods can't be scheduled to the same node")).To(Equal(true)) }, }, { - name: "ordinal 0 is scheduled to kube-node-3, get 0 node, ordinal 1 should be scheduled to none", - ordinal: 1, - podFn: newHAPDPod, - nodesFn: fakeZeroNode, - podListFn: podListFn(map[string][]int32{"kube-node-3": []int32{0}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "ordinal 0 is scheduled to kube-node-3, get 0 node, ordinal 1 should be scheduled to none", + ordinal: 1, + podFn: newHAPDPod, + nodesFn: fakeZeroNode, + podListFn: podListFn(map[string][]int32{"kube-node-3": []int32{0}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).To(HaveOccurred()) g.Expect(strings.Contains(err.Error(), "no suitable node for pod: default/demo-pd-1")).To(Equal(true)) }, }, { - name: "ordinal 0 is scheduled to kube-node-2, ordinal 1 is kube-node-3, ordinal 2 should be scheduled to kube-node-1", - ordinal: 2, - podFn: newHAPDPod, - nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "ordinal 0 is scheduled to kube-node-2, ordinal 1 is kube-node-3, ordinal 2 should be scheduled to kube-node-1", + ordinal: 2, + podFn: newHAPDPod, + nodesFn: fakeThreeNodes, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(1)) @@ -203,39 +181,33 @@ func TestHAFilter(t *testing.T) { }, }, { - name: "ordinal 0 is scheduled to kube-node-1, ordinal 1 is kube-node-3, ordinal 2 should be scheduled to kube-node-1", - ordinal: 2, - podFn: newHAPDPod, - nodesFn: fakeTwoNodes, - podListFn: podListFn(map[string][]int32{"kube-node-1": []int32{0}, "kube-node-3": []int32{1}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "ordinal 0 is scheduled to kube-node-1, ordinal 1 is kube-node-3, ordinal 2 should be scheduled to kube-node-1", + ordinal: 2, + podFn: newHAPDPod, + nodesFn: fakeTwoNodes, + podListFn: podListFn(map[string][]int32{"kube-node-1": []int32{0}, "kube-node-3": []int32{1}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).To(HaveOccurred()) - g.Expect(strings.Contains(err.Error(), "the first 3 pods can't scheduled to the same node")).To(Equal(true)) + g.Expect(strings.Contains(err.Error(), "the first 3 pods can't be scheduled to the same node")).To(Equal(true)) }, }, { - name: "ordinal 0 is scheduled to kube-node-1, ordinal 1 is kube-node-3, get 0 node, ordinal 2 should be scheduled to none", - ordinal: 2, - podFn: newHAPDPod, - nodesFn: fakeZeroNode, - podListFn: podListFn(map[string][]int32{"kube-node-1": []int32{0}, "kube-node-3": []int32{1}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "ordinal 0 is scheduled to kube-node-1, ordinal 1 is kube-node-3, get 0 node, ordinal 2 should be scheduled to none", + ordinal: 2, + podFn: newHAPDPod, + nodesFn: fakeZeroNode, + podListFn: podListFn(map[string][]int32{"kube-node-1": []int32{0}, "kube-node-3": []int32{1}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).To(HaveOccurred()) g.Expect(strings.Contains(err.Error(), "no suitable node for pod: default/demo-pd-2")).To(Equal(true)) }, }, { - name: "the first three oridnals get to 3 nodes, the ordinal 3 should scheduled to 1,2,3", - ordinal: 3, - podFn: newHAPDPod, - nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}, "kube-node-1": []int32{2}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "the first three oridnals get to 3 nodes, the ordinal 3 should scheduled to 1,2,3", + ordinal: 3, + podFn: newHAPDPod, + nodesFn: fakeThreeNodes, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}, "kube-node-1": []int32{2}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(3)) @@ -243,13 +215,11 @@ func TestHAFilter(t *testing.T) { }, }, { - name: "the first four oridnals get to 3 nodes, the ordinal 4 should scheduled to 2,3", - ordinal: 4, - podFn: newHAPDPod, - nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}, "kube-node-1": []int32{2, 3}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "the first four oridnals get to 3 nodes, the ordinal 4 should scheduled to 2,3", + ordinal: 4, + podFn: newHAPDPod, + nodesFn: fakeThreeNodes, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}, "kube-node-1": []int32{2, 3}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(2)) @@ -257,13 +227,11 @@ func TestHAFilter(t *testing.T) { }, }, { - name: "the first five oridnals get to 3 nodes, the ordinal 5 should scheduled to 2", - ordinal: 5, - podFn: newHAPDPod, - nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "the first five oridnals get to 3 nodes, the ordinal 5 should scheduled to 2", + ordinal: 5, + podFn: newHAPDPod, + nodesFn: fakeThreeNodes, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(1)) @@ -271,13 +239,11 @@ func TestHAFilter(t *testing.T) { }, }, { - name: "the first five oridnals get to 3 nodes, got 2 nodes(no node-2), the ordinal 5 should scheduled to 1,3", - ordinal: 5, - podFn: newHAPDPod, - nodesFn: fakeTwoNodes, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "the first five oridnals get to 3 nodes, got 2 nodes(no node-2), the ordinal 5 should scheduled to 1,3", + ordinal: 5, + podFn: newHAPDPod, + nodesFn: fakeTwoNodes, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(2)) @@ -285,13 +251,11 @@ func TestHAFilter(t *testing.T) { }, }, { - name: "the first five oridnals get to 3 nodes, got 1 nodes(no node-2 node-1), the ordinal 5 should scheduled to 3", - ordinal: 5, - podFn: newHAPDPod, - nodesFn: fakeOneNode, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "the first five oridnals get to 3 nodes, got 1 nodes(no node-2 node-1), the ordinal 5 should scheduled to 3", + ordinal: 5, + podFn: newHAPDPod, + nodesFn: fakeOneNode, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(1)) @@ -299,26 +263,22 @@ func TestHAFilter(t *testing.T) { }, }, { - name: "the first five oridnals get to 3 nodes, got 0 nodes, the ordinal 5 should scheduled to none", - ordinal: 5, - podFn: newHAPDPod, - nodesFn: fakeZeroNode, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "the first five oridnals get to 3 nodes, got 0 nodes, the ordinal 5 should scheduled to none", + ordinal: 5, + podFn: newHAPDPod, + nodesFn: fakeZeroNode, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).To(HaveOccurred()) g.Expect(strings.Contains(err.Error(), "no suitable node for pod: default/demo-pd-5")).To(Equal(true)) }, }, { - name: "the first six oridnals get to 3 nodes, the ordinal 6 should scheduled to 1,2,3", - ordinal: 6, - podFn: newHAPDPod, - nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0, 5}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), - pdReplicas: 3, - tikvReplicas: 3, + name: "the first six oridnals get to 3 nodes, the ordinal 6 should scheduled to 1,2,3", + ordinal: 6, + podFn: newHAPDPod, + nodesFn: fakeThreeNodes, + podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0, 5}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(3)) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 043ffd0d0e9..abeb59eaf1a 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -44,17 +44,17 @@ type scheduler struct { } // NewScheduler returns a Scheduler -func NewScheduler(kubeCli kubernetes.Interface, pdReplicas, tikvReplicas int32) Scheduler { +func NewScheduler(kubeCli kubernetes.Interface) Scheduler { return &scheduler{ kubeCli: kubeCli, predicates: []predicates.Predicate{ - predicates.NewHA(kubeCli, pdReplicas, tikvReplicas), + predicates.NewHA(kubeCli), }, } } // Filter selects a set of nodes from *schedulerapiv1.ExtenderArgs.Nodes when this is a pd or tikv pod -// else return the original nodes. +// otherwise, returns the original nodes. func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.ExtenderFilterResult, error) { pod := &args.Pod ns := pod.GetNamespace() @@ -64,24 +64,25 @@ func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.E var clusterName string var exist bool if clusterName, exist = pod.Labels[label.InstanceLabelKey]; !exist { - return nil, fmt.Errorf("can't find clusterName in pod labels: %s/%s", ns, podName) + return &schedulerapiv1.ExtenderFilterResult{ + Nodes: args.Nodes, + }, fmt.Errorf("can't find clusterName in pod labels: %s/%s", ns, podName) } if component := pod.Labels[label.ComponentLabelKey]; component != label.PDLabelVal && component != label.TiKVLabelVal { return &schedulerapiv1.ExtenderFilterResult{ - Nodes: args.Nodes, - NodeNames: args.NodeNames, + Nodes: args.Nodes, }, nil } glog.Infof("scheduling pod: %s/%s", ns, podName) var err error for _, predicate := range s.predicates { - glog.Infof("entering predicate: %s, nodes: %v", predicate.Name(), getNodeNames(kubeNodes)) + glog.V(4).Infof("entering predicate: %s, nodes: %v", predicate.Name(), getNodeNames(kubeNodes)) kubeNodes, err = predicate.Filter(clusterName, pod, kubeNodes) if err != nil { return nil, err } - glog.Infof("leaving predicate: %s, nodes: %v", predicate.Name(), getNodeNames(kubeNodes)) + glog.V(4).Infof("leaving predicate: %s, nodes: %v", predicate.Name(), getNodeNames(kubeNodes)) } return &schedulerapiv1.ExtenderFilterResult{ @@ -89,7 +90,7 @@ func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.E }, nil } -// We didn't pass `prioritizeVerb` to kubernetes scheduler extender's config file, this method will not be called. +// We don't pass `prioritizeVerb` to kubernetes scheduler extender's config file, this method will not be called. func (s *scheduler) Priority(args *schedulerapiv1.ExtenderArgs) (schedulerapiv1.HostPriorityList, error) { result := schedulerapiv1.HostPriorityList{} diff --git a/pkg/scheduler/server/mux.go b/pkg/scheduler/server/mux.go index d74d6719ddc..b31d0723fff 100644 --- a/pkg/scheduler/server/mux.go +++ b/pkg/scheduler/server/mux.go @@ -36,8 +36,8 @@ type server struct { } // StartServer starts a kubernetes scheduler extender http apiserver -func StartServer(kubeCli kubernetes.Interface, port int, pdReplicas, tikvReplicas int32) { - s := scheduler.NewScheduler(kubeCli, pdReplicas, tikvReplicas) +func StartServer(kubeCli kubernetes.Interface, port int) { + s := scheduler.NewScheduler(kubeCli) svr := &server{scheduler: s} ws := new(restful.WebService) From 27f0a8d29e79be56302f2778535506d2a587576f Mon Sep 17 00:00:00 2001 From: weekface Date: Tue, 30 Oct 2018 14:58:57 +0800 Subject: [PATCH 06/12] address comment --- pkg/scheduler/scheduler.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index abeb59eaf1a..7cfcf177d92 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -14,8 +14,6 @@ package scheduler import ( - "fmt" - "github.com/golang/glog" "github.com/pingcap/tidb-operator/pkg/label" "github.com/pingcap/tidb-operator/pkg/scheduler/predicates" @@ -64,9 +62,10 @@ func (s *scheduler) Filter(args *schedulerapiv1.ExtenderArgs) (*schedulerapiv1.E var clusterName string var exist bool if clusterName, exist = pod.Labels[label.InstanceLabelKey]; !exist { + glog.Warningf("can't find clusterName in pod labels: %s/%s", ns, podName) return &schedulerapiv1.ExtenderFilterResult{ Nodes: args.Nodes, - }, fmt.Errorf("can't find clusterName in pod labels: %s/%s", ns, podName) + }, nil } if component := pod.Labels[label.ComponentLabelKey]; component != label.PDLabelVal && component != label.TiKVLabelVal { return &schedulerapiv1.ExtenderFilterResult{ From b7e8d1ed9c9acbc40ff15e978e544571453d2c84 Mon Sep 17 00:00:00 2001 From: weekface Date: Tue, 30 Oct 2018 17:52:16 +0800 Subject: [PATCH 07/12] address comment --- README.md | 2 +- .../tidb-operator/files/scheduler-policy.json | 38 ++++++++++++++++++ .../templates/scheduler-policy-configmap.yaml | 39 +------------------ 3 files changed, 40 insertions(+), 39 deletions(-) create mode 100644 charts/tidb-operator/files/scheduler-policy.json diff --git a/README.md b/README.md index f9b076400e4..dfa9db21b11 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ TiDB Operator manages [TiDB](https://github.com/pingcap/tidb) clusters on [Kuber Users can deploy and manage multiple TiDB clusters on a single Kubernetes cluster easily. -- __Automatic failover__ (WIP) +- __Automatic failover__ TiDB Operator automatically performs failover for your TiDB cluster when node failures occur. diff --git a/charts/tidb-operator/files/scheduler-policy.json b/charts/tidb-operator/files/scheduler-policy.json new file mode 100644 index 00000000000..53ca0bdff27 --- /dev/null +++ b/charts/tidb-operator/files/scheduler-policy.json @@ -0,0 +1,38 @@ +{ + "kind" : "Policy", + "apiVersion" : "v1", + "predicates": [ + {"name": "MatchInterPodAffinity"}, + {"name": "CheckVolumeBinding"}, + {"name": "CheckNodeCondition"}, + {"name": "GeneralPredicates"}, + {"name": "HostName"}, + {"name": "PodFitsHostPorts"}, + {"name": "MatchNodeSelector"}, + {"name": "PodFitsResources"}, + {"name": "NoDiskConflict"}, + {"name": "PodToleratesNodeTaints"}, + {"name": "CheckNodeMemoryPressure"}, + {"name": "CheckNodeDiskPressure"} + ], + "priorities": [ + {"name": "EqualPriority", "weight": 1}, + {"name": "ImageLocalityPriority", "weight": 1}, + {"name": "LeastRequestedPriority", "weight": 1}, + {"name": "BalancedResourceAllocation", "weight": 1}, + {"name": "SelectorSpreadPriority", "weight": 1}, + {"name": "NodePreferAvoidPodsPriority", "weight": 1}, + {"name": "NodeAffinityPriority", "weight": 1}, + {"name": "TaintTolerationPriority", "weight": 1}, + {"name": "MostRequestedPriority", "weight": 1} + ], + "extenders": [ + { + "urlPrefix": "http://127.0.0.1:10262/scheduler", + "filterVerb": "filter", + "weight": 1, + "httpTimeout": 30000000000, + "enableHttps": false + } + ] +} \ No newline at end of file diff --git a/charts/tidb-operator/templates/scheduler-policy-configmap.yaml b/charts/tidb-operator/templates/scheduler-policy-configmap.yaml index 1524ca36a21..75a6fb5a84a 100644 --- a/charts/tidb-operator/templates/scheduler-policy-configmap.yaml +++ b/charts/tidb-operator/templates/scheduler-policy-configmap.yaml @@ -10,41 +10,4 @@ metadata: helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} data: policy.cfg: |- - { - "kind" : "Policy", - "apiVersion" : "v1", - "predicates": [ - {"name": "MatchInterPodAffinity"}, - {"name": "CheckVolumeBinding"}, - {"name": "CheckNodeCondition"}, - {"name": "GeneralPredicates"}, - {"name": "HostName"}, - {"name": "PodFitsHostPorts"}, - {"name": "MatchNodeSelector"}, - {"name": "PodFitsResources"}, - {"name": "NoDiskConflict"}, - {"name": "PodToleratesNodeTaints"}, - {"name": "CheckNodeMemoryPressure"}, - {"name": "CheckNodeDiskPressure"} - ], - "priorities": [ - {"name": "EqualPriority", "weight": 1}, - {"name": "ImageLocalityPriority", "weight": 1}, - {"name": "LeastRequestedPriority", "weight": 1}, - {"name": "BalancedResourceAllocation", "weight": 1}, - {"name": "SelectorSpreadPriority", "weight": 1}, - {"name": "NodePreferAvoidPodsPriority", "weight": 1}, - {"name": "NodeAffinityPriority", "weight": 1}, - {"name": "TaintTolerationPriority", "weight": 1}, - {"name": "MostRequestedPriority", "weight": 1} - ], - "extenders": [ - { - "urlPrefix": "http://127.0.0.1:10262/scheduler", - "filterVerb": "filter", - "weight": 1, - "httpTimeout": 30000000000, - "enableHttps": false - } - ] - } +{{ .Files.Get "files/scheduler-policy.json" | indent 4 }} From 2675d6ff46bcf69653a8551d5354989cd4152314 Mon Sep 17 00:00:00 2001 From: weekface Date: Tue, 30 Oct 2018 20:03:26 +0800 Subject: [PATCH 08/12] make gofmt happy --- pkg/manager/member/pd_failover_test.go | 168 +++++++++--------- pkg/manager/member/pd_member_manager_test.go | 62 +++---- .../member/tidb_member_manager_test.go | 12 +- .../member/tikv_member_manager_test.go | 54 +++--- pkg/scheduler/predicates/ha_test.go | 28 +-- 5 files changed, 162 insertions(+), 162 deletions(-) diff --git a/pkg/manager/member/pd_failover_test.go b/pkg/manager/member/pd_failover_test.go index 62efeca9f73..bbb97183095 100644 --- a/pkg/manager/member/pd_failover_test.go +++ b/pkg/manager/member/pd_failover_test.go @@ -94,11 +94,11 @@ func TestPDFailoverFailover(t *testing.T) { } tests := []testcase{ { - name: "all members are ready", - update: allMembersReady, - hasPVC: true, - hasPV: true, - hasPod: true, + name: "all members are ready", + update: allMembersReady, + hasPVC: true, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, @@ -111,11 +111,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "two members are not ready, not in quorum", - update: twoMembersNotReady, - hasPVC: true, - hasPV: true, - hasPod: true, + name: "two members are not ready, not in quorum", + update: twoMembersNotReady, + hasPVC: true, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, @@ -131,11 +131,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "two members are ready and a failure member", - update: oneFailureMember, - hasPVC: true, - hasPV: true, - hasPod: true, + name: "two members are ready and a failure member", + update: oneFailureMember, + hasPVC: true, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, @@ -160,9 +160,9 @@ func TestPDFailoverFailover(t *testing.T) { pd1.LastTransitionTime = metav1.Time{Time: time.Now().Add(-2 * time.Minute)} tc.Status.PD.Members[pd1Name] = pd1 }, - hasPVC: true, - hasPV: true, - hasPod: true, + hasPVC: true, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, @@ -183,9 +183,9 @@ func TestPDFailoverFailover(t *testing.T) { pd1.LastTransitionTime = metav1.Time{} tc.Status.PD.Members[pd1Name] = pd1 }, - hasPVC: true, - hasPV: true, - hasPod: true, + hasPVC: true, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, @@ -198,11 +198,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "has one not ready member, and exceed deadline, don't have PVC, have PV", - update: oneNotReadyMember, - hasPVC: true, - hasPV: false, - hasPod: true, + name: "has one not ready member, and exceed deadline, don't have PVC, have PV", + update: oneNotReadyMember, + hasPVC: true, + hasPV: false, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, @@ -215,11 +215,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "has one not ready member, and exceed deadline, have PVC, don't have PV", - update: oneNotReadyMember, - hasPVC: true, - hasPV: false, - hasPod: true, + name: "has one not ready member, and exceed deadline, have PVC, don't have PV", + update: oneNotReadyMember, + hasPVC: true, + hasPV: false, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, @@ -232,11 +232,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "has one not ready member, and exceed deadline, return requeue error", - update: oneNotReadyMember, - hasPVC: true, - hasPV: true, - hasPod: true, + name: "has one not ready member, and exceed deadline, return requeue error", + update: oneNotReadyMember, + hasPVC: true, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, @@ -253,11 +253,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "has one not ready member, and exceed deadline, don't have PVC, has Pod, delete pod success", - update: oneNotReadyMemberAndAFailureMember, - hasPVC: false, - hasPV: true, - hasPod: true, + name: "has one not ready member, and exceed deadline, don't have PVC, has Pod, delete pod success", + update: oneNotReadyMemberAndAFailureMember, + hasPVC: false, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, @@ -274,11 +274,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "has one not ready member, and exceed deadline, don't have PVC, has Pod, delete pod failed", - update: oneNotReadyMemberAndAFailureMember, - hasPVC: false, - hasPV: true, - hasPod: true, + name: "has one not ready member, and exceed deadline, don't have PVC, has Pod, delete pod failed", + update: oneNotReadyMemberAndAFailureMember, + hasPVC: false, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: true, @@ -295,11 +295,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "one failure member, delete member failed", - update: oneFailureMember, - hasPVC: true, - hasPV: true, - hasPod: true, + name: "one failure member, delete member failed", + update: oneFailureMember, + hasPVC: true, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: true, delPodFailed: false, @@ -315,11 +315,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "one failure member, don't have pvc, have pod with deletetimestamp", - update: oneNotReadyMemberAndAFailureMember, - hasPVC: false, - hasPV: true, - hasPod: true, + name: "one failure member, don't have pvc, have pod with deletetimestamp", + update: oneNotReadyMemberAndAFailureMember, + hasPVC: false, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: true, delMemberFailed: false, delPodFailed: false, @@ -335,11 +335,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "one failure member, don't have pvc, have pod without deletetimestamp, delete pod success", - update: oneNotReadyMemberAndAFailureMember, - hasPVC: false, - hasPV: true, - hasPod: true, + name: "one failure member, don't have pvc, have pod without deletetimestamp, delete pod success", + update: oneNotReadyMemberAndAFailureMember, + hasPVC: false, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, @@ -355,11 +355,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "one failure member, don't have pvc, have pod without deletetimestamp, delete pod failed", - update: oneNotReadyMemberAndAFailureMember, - hasPVC: false, - hasPV: true, - hasPod: true, + name: "one failure member, don't have pvc, have pod without deletetimestamp, delete pod failed", + update: oneNotReadyMemberAndAFailureMember, + hasPVC: false, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: true, @@ -375,11 +375,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "one failure member, don't have pv", - update: oneNotReadyMemberAndAFailureMember, - hasPVC: true, - hasPV: false, - hasPod: true, + name: "one failure member, don't have pv", + update: oneNotReadyMemberAndAFailureMember, + hasPVC: true, + hasPV: false, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, @@ -403,9 +403,9 @@ func TestPDFailoverFailover(t *testing.T) { pd1.PVUID = "xxx" tc.Status.PD.FailureMembers[pd1Name] = pd1 }, - hasPVC: true, - hasPV: true, - hasPod: true, + hasPVC: true, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, @@ -421,11 +421,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "one failure members, pvc delete fail", - update: oneNotReadyMemberAndAFailureMember, - hasPVC: true, - hasPV: true, - hasPod: true, + name: "one failure members, pvc delete fail", + update: oneNotReadyMemberAndAFailureMember, + hasPVC: true, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, @@ -441,11 +441,11 @@ func TestPDFailoverFailover(t *testing.T) { }, }, { - name: "pd status sync failed", - update: allMembersReady, - hasPVC: true, - hasPV: true, - hasPod: true, + name: "pd status sync failed", + update: allMembersReady, + hasPVC: true, + hasPV: true, + hasPod: true, podWithDeletionTimestamp: false, delMemberFailed: false, delPodFailed: false, diff --git a/pkg/manager/member/pd_member_manager_test.go b/pkg/manager/member/pd_member_manager_test.go index fb051d05bf0..4550bbb4f26 100644 --- a/pkg/manager/member/pd_member_manager_test.go +++ b/pkg/manager/member/pd_member_manager_test.go @@ -110,10 +110,10 @@ func TestPDMemberManagerSyncCreate(t *testing.T) { errWhenCreateStatefulSet: false, errWhenCreatePDService: false, errWhenCreatePDPeerService: false, - err: false, - pdSvcCreated: true, - pdPeerSvcCreated: true, - setCreated: true, + err: false, + pdSvcCreated: true, + pdPeerSvcCreated: true, + setCreated: true, }, { name: "tidbcluster's storage format is wrong", @@ -123,10 +123,10 @@ func TestPDMemberManagerSyncCreate(t *testing.T) { errWhenCreateStatefulSet: false, errWhenCreatePDService: false, errWhenCreatePDPeerService: false, - err: true, - pdSvcCreated: true, - pdPeerSvcCreated: true, - setCreated: false, + err: true, + pdSvcCreated: true, + pdPeerSvcCreated: true, + setCreated: false, }, { name: "error when create statefulset", @@ -134,10 +134,10 @@ func TestPDMemberManagerSyncCreate(t *testing.T) { errWhenCreateStatefulSet: true, errWhenCreatePDService: false, errWhenCreatePDPeerService: false, - err: true, - pdSvcCreated: true, - pdPeerSvcCreated: true, - setCreated: false, + err: true, + pdSvcCreated: true, + pdPeerSvcCreated: true, + setCreated: false, }, { name: "error when create pd service", @@ -145,10 +145,10 @@ func TestPDMemberManagerSyncCreate(t *testing.T) { errWhenCreateStatefulSet: false, errWhenCreatePDService: true, errWhenCreatePDPeerService: false, - err: true, - pdSvcCreated: false, - pdPeerSvcCreated: false, - setCreated: false, + err: true, + pdSvcCreated: false, + pdPeerSvcCreated: false, + setCreated: false, }, { name: "error when create pd peer service", @@ -156,10 +156,10 @@ func TestPDMemberManagerSyncCreate(t *testing.T) { errWhenCreateStatefulSet: false, errWhenCreatePDService: false, errWhenCreatePDPeerService: true, - err: true, - pdSvcCreated: true, - pdPeerSvcCreated: false, - setCreated: false, + err: true, + pdSvcCreated: true, + pdPeerSvcCreated: false, + setCreated: false, }, } @@ -313,10 +313,10 @@ func TestPDMemberManagerSyncUpdate(t *testing.T) { errWhenUpdateStatefulSet: false, errWhenUpdatePDService: false, errWhenUpdatePDPeerService: false, - err: true, - expectPDServiceFn: nil, - expectPDPeerServiceFn: nil, - expectStatefulSetFn: nil, + err: true, + expectPDServiceFn: nil, + expectPDPeerServiceFn: nil, + expectStatefulSetFn: nil, }, { name: "error when update pd service", @@ -333,10 +333,10 @@ func TestPDMemberManagerSyncUpdate(t *testing.T) { errWhenUpdateStatefulSet: false, errWhenUpdatePDService: true, errWhenUpdatePDPeerService: false, - err: true, - expectPDServiceFn: nil, - expectPDPeerServiceFn: nil, - expectStatefulSetFn: nil, + err: true, + expectPDServiceFn: nil, + expectPDPeerServiceFn: nil, + expectStatefulSetFn: nil, }, { name: "error when update statefulset", @@ -351,9 +351,9 @@ func TestPDMemberManagerSyncUpdate(t *testing.T) { errWhenUpdateStatefulSet: true, errWhenUpdatePDService: false, errWhenUpdatePDPeerService: false, - err: true, - expectPDServiceFn: nil, - expectPDPeerServiceFn: nil, + err: true, + expectPDServiceFn: nil, + expectPDPeerServiceFn: nil, expectStatefulSetFn: func(g *GomegaWithT, set *apps.StatefulSet, err error) { g.Expect(err).NotTo(HaveOccurred()) }, diff --git a/pkg/manager/member/tidb_member_manager_test.go b/pkg/manager/member/tidb_member_manager_test.go index 144adb4784a..9e6ebff59a1 100644 --- a/pkg/manager/member/tidb_member_manager_test.go +++ b/pkg/manager/member/tidb_member_manager_test.go @@ -79,15 +79,15 @@ func TestTiDBMemberManagerSyncCreate(t *testing.T) { name: "normal", prepare: nil, errWhenCreateStatefulSet: false, - err: false, - setCreated: true, + err: false, + setCreated: true, }, { name: "error when create statefulset", prepare: nil, errWhenCreateStatefulSet: true, - err: true, - setCreated: false, + err: true, + setCreated: false, }, } @@ -162,7 +162,7 @@ func TestTiDBMemberManagerSyncUpdate(t *testing.T) { tc.Status.TiKV.Phase = v1alpha1.NormalPhase }, errWhenUpdateStatefulSet: false, - err: false, + err: false, expectStatefulSetFn: func(g *GomegaWithT, set *apps.StatefulSet, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(int(*set.Spec.Replicas)).To(Equal(5)) @@ -176,7 +176,7 @@ func TestTiDBMemberManagerSyncUpdate(t *testing.T) { tc.Status.TiKV.Phase = v1alpha1.NormalPhase }, errWhenUpdateStatefulSet: true, - err: true, + err: true, expectStatefulSetFn: func(g *GomegaWithT, set *apps.StatefulSet, err error) { g.Expect(err).NotTo(HaveOccurred()) }, diff --git a/pkg/manager/member/tikv_member_manager_test.go b/pkg/manager/member/tikv_member_manager_test.go index 0849fec2e8f..c805ad47a6b 100644 --- a/pkg/manager/member/tikv_member_manager_test.go +++ b/pkg/manager/member/tikv_member_manager_test.go @@ -111,11 +111,11 @@ func TestTiKVMemberManagerSyncCreate(t *testing.T) { errWhenCreateStatefulSet: false, errWhenCreateTiKVPeerService: false, - err: false, - tikvPeerSvcCreated: true, - setCreated: true, - pdStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, - tombstoneStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, + err: false, + tikvPeerSvcCreated: true, + setCreated: true, + pdStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, + tombstoneStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, }, { name: "tidbcluster's storage format is wrong", @@ -124,33 +124,33 @@ func TestTiKVMemberManagerSyncCreate(t *testing.T) { }, errWhenCreateStatefulSet: false, errWhenCreateTiKVPeerService: false, - err: true, - tikvPeerSvcCreated: true, - setCreated: false, - pdStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, - tombstoneStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, + err: true, + tikvPeerSvcCreated: true, + setCreated: false, + pdStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, + tombstoneStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, }, { name: "error when create statefulset", prepare: nil, errWhenCreateStatefulSet: true, errWhenCreateTiKVPeerService: false, - err: true, - tikvPeerSvcCreated: true, - setCreated: false, - pdStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, - tombstoneStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, + err: true, + tikvPeerSvcCreated: true, + setCreated: false, + pdStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, + tombstoneStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, }, { name: "error when create tikv peer service", prepare: nil, errWhenCreateStatefulSet: false, errWhenCreateTiKVPeerService: true, - err: true, - tikvPeerSvcCreated: false, - setCreated: false, - pdStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, - tombstoneStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, + err: true, + tikvPeerSvcCreated: false, + setCreated: false, + pdStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, + tombstoneStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, }, } @@ -266,7 +266,7 @@ func TestTiKVMemberManagerSyncUpdate(t *testing.T) { errWhenUpdateTiKVPeerService: false, errWhenGetStores: false, err: false, - expectTiKVPeerServiceFn: nil, + expectTiKVPeerServiceFn: nil, expectStatefulSetFn: func(g *GomegaWithT, set *apps.StatefulSet, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(int(*set.Spec.Replicas)).To(Equal(4)) @@ -287,9 +287,9 @@ func TestTiKVMemberManagerSyncUpdate(t *testing.T) { tombstoneStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, errWhenUpdateStatefulSet: false, errWhenUpdateTiKVPeerService: false, - err: true, - expectTiKVPeerServiceFn: nil, - expectStatefulSetFn: nil, + err: true, + expectTiKVPeerServiceFn: nil, + expectStatefulSetFn: nil, }, { name: "error when update statefulset", @@ -301,8 +301,8 @@ func TestTiKVMemberManagerSyncUpdate(t *testing.T) { tombstoneStores: &controller.StoresInfo{Count: 0, Stores: []*controller.StoreInfo{}}, errWhenUpdateStatefulSet: true, errWhenUpdateTiKVPeerService: false, - err: true, - expectTiKVPeerServiceFn: nil, + err: true, + expectTiKVPeerServiceFn: nil, expectStatefulSetFn: func(g *GomegaWithT, set *apps.StatefulSet, err error) { g.Expect(err).NotTo(HaveOccurred()) }, @@ -319,7 +319,7 @@ func TestTiKVMemberManagerSyncUpdate(t *testing.T) { errWhenUpdateTiKVPeerService: false, errWhenGetStores: true, err: true, - expectTiKVPeerServiceFn: nil, + expectTiKVPeerServiceFn: nil, expectStatefulSetFn: func(g *GomegaWithT, set *apps.StatefulSet, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(int(*set.Spec.Replicas)).To(Equal(3)) diff --git a/pkg/scheduler/predicates/ha_test.go b/pkg/scheduler/predicates/ha_test.go index 9ba056cbda1..c0a80080f9a 100644 --- a/pkg/scheduler/predicates/ha_test.go +++ b/pkg/scheduler/predicates/ha_test.go @@ -117,7 +117,7 @@ func TestHAFilter(t *testing.T) { ordinal: 1, podFn: newHAPDPod, nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"": []int32{0}}), + podListFn: podListFn(map[string][]int32{"": {0}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(strings.Contains(err.Error(), "waiting for pod: default/demo-pd-0")).To(Equal(true)) }, @@ -139,7 +139,7 @@ func TestHAFilter(t *testing.T) { ordinal: 1, podFn: newHAPDPod, nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"kube-node-1": []int32{0}}), + podListFn: podListFn(map[string][]int32{"kube-node-1": {0}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(2)) @@ -151,7 +151,7 @@ func TestHAFilter(t *testing.T) { ordinal: 1, podFn: newHAPDPod, nodesFn: fakeOneNode, - podListFn: podListFn(map[string][]int32{"kube-node-3": []int32{0}}), + podListFn: podListFn(map[string][]int32{"kube-node-3": {0}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).To(HaveOccurred()) g.Expect(strings.Contains(err.Error(), "the first 3 pods can't be scheduled to the same node")).To(Equal(true)) @@ -162,7 +162,7 @@ func TestHAFilter(t *testing.T) { ordinal: 1, podFn: newHAPDPod, nodesFn: fakeZeroNode, - podListFn: podListFn(map[string][]int32{"kube-node-3": []int32{0}}), + podListFn: podListFn(map[string][]int32{"kube-node-3": {0}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).To(HaveOccurred()) g.Expect(strings.Contains(err.Error(), "no suitable node for pod: default/demo-pd-1")).To(Equal(true)) @@ -173,7 +173,7 @@ func TestHAFilter(t *testing.T) { ordinal: 2, podFn: newHAPDPod, nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}}), + podListFn: podListFn(map[string][]int32{"kube-node-2": {0}, "kube-node-3": {1}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(1)) @@ -185,7 +185,7 @@ func TestHAFilter(t *testing.T) { ordinal: 2, podFn: newHAPDPod, nodesFn: fakeTwoNodes, - podListFn: podListFn(map[string][]int32{"kube-node-1": []int32{0}, "kube-node-3": []int32{1}}), + podListFn: podListFn(map[string][]int32{"kube-node-1": {0}, "kube-node-3": {1}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).To(HaveOccurred()) g.Expect(strings.Contains(err.Error(), "the first 3 pods can't be scheduled to the same node")).To(Equal(true)) @@ -196,7 +196,7 @@ func TestHAFilter(t *testing.T) { ordinal: 2, podFn: newHAPDPod, nodesFn: fakeZeroNode, - podListFn: podListFn(map[string][]int32{"kube-node-1": []int32{0}, "kube-node-3": []int32{1}}), + podListFn: podListFn(map[string][]int32{"kube-node-1": {0}, "kube-node-3": {1}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).To(HaveOccurred()) g.Expect(strings.Contains(err.Error(), "no suitable node for pod: default/demo-pd-2")).To(Equal(true)) @@ -207,7 +207,7 @@ func TestHAFilter(t *testing.T) { ordinal: 3, podFn: newHAPDPod, nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}, "kube-node-1": []int32{2}}), + podListFn: podListFn(map[string][]int32{"kube-node-2": {0}, "kube-node-3": {1}, "kube-node-1": {2}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(3)) @@ -219,7 +219,7 @@ func TestHAFilter(t *testing.T) { ordinal: 4, podFn: newHAPDPod, nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1}, "kube-node-1": []int32{2, 3}}), + podListFn: podListFn(map[string][]int32{"kube-node-2": {0}, "kube-node-3": {1}, "kube-node-1": {2, 3}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(2)) @@ -231,7 +231,7 @@ func TestHAFilter(t *testing.T) { ordinal: 5, podFn: newHAPDPod, nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), + podListFn: podListFn(map[string][]int32{"kube-node-2": {0}, "kube-node-3": {1, 4}, "kube-node-1": {2, 3}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(1)) @@ -243,7 +243,7 @@ func TestHAFilter(t *testing.T) { ordinal: 5, podFn: newHAPDPod, nodesFn: fakeTwoNodes, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), + podListFn: podListFn(map[string][]int32{"kube-node-2": {0}, "kube-node-3": {1, 4}, "kube-node-1": {2, 3}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(2)) @@ -255,7 +255,7 @@ func TestHAFilter(t *testing.T) { ordinal: 5, podFn: newHAPDPod, nodesFn: fakeOneNode, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), + podListFn: podListFn(map[string][]int32{"kube-node-2": {0}, "kube-node-3": {1, 4}, "kube-node-1": {2, 3}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(1)) @@ -267,7 +267,7 @@ func TestHAFilter(t *testing.T) { ordinal: 5, podFn: newHAPDPod, nodesFn: fakeZeroNode, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), + podListFn: podListFn(map[string][]int32{"kube-node-2": {0}, "kube-node-3": {1, 4}, "kube-node-1": {2, 3}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).To(HaveOccurred()) g.Expect(strings.Contains(err.Error(), "no suitable node for pod: default/demo-pd-5")).To(Equal(true)) @@ -278,7 +278,7 @@ func TestHAFilter(t *testing.T) { ordinal: 6, podFn: newHAPDPod, nodesFn: fakeThreeNodes, - podListFn: podListFn(map[string][]int32{"kube-node-2": []int32{0, 5}, "kube-node-3": []int32{1, 4}, "kube-node-1": []int32{2, 3}}), + podListFn: podListFn(map[string][]int32{"kube-node-2": {0, 5}, "kube-node-3": {1, 4}, "kube-node-1": {2, 3}}), expectFn: func(nodes []apiv1.Node, err error) { g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(nodes)).To(Equal(3)) From e42c89ed11325d2dfe4d696ed11facde0a8e5d8e Mon Sep 17 00:00:00 2001 From: weekface Date: Wed, 31 Oct 2018 10:48:27 +0800 Subject: [PATCH 09/12] update vendor --- Gopkg.lock | 15 + Gopkg.toml | 4 + .../github.com/emicklei/go-restful/.gitignore | 70 ++++ .../emicklei/go-restful/.travis.yml | 6 + .../github.com/emicklei/go-restful/CHANGES.md | 242 ++++++++++++ vendor/github.com/emicklei/go-restful/LICENSE | 22 ++ .../github.com/emicklei/go-restful/Makefile | 7 + .../github.com/emicklei/go-restful/README.md | 88 +++++ vendor/github.com/emicklei/go-restful/Srcfile | 1 + .../emicklei/go-restful/bench_curly_test.go | 51 +++ .../emicklei/go-restful/bench_test.go | 43 ++ .../emicklei/go-restful/bench_test.sh | 10 + .../emicklei/go-restful/compress.go | 123 ++++++ .../emicklei/go-restful/compress_test.go | 125 ++++++ .../emicklei/go-restful/compressor_cache.go | 103 +++++ .../emicklei/go-restful/compressor_pools.go | 91 +++++ .../emicklei/go-restful/compressors.go | 54 +++ .../emicklei/go-restful/constants.go | 30 ++ .../emicklei/go-restful/container.go | 371 ++++++++++++++++++ .../emicklei/go-restful/container_test.go | 83 ++++ .../emicklei/go-restful/cors_filter.go | 202 ++++++++++ .../emicklei/go-restful/cors_filter_test.go | 129 ++++++ .../emicklei/go-restful/coverage.sh | 2 + .../github.com/emicklei/go-restful/curly.go | 164 ++++++++ .../emicklei/go-restful/curly_route.go | 52 +++ .../emicklei/go-restful/curly_test.go | 238 +++++++++++ vendor/github.com/emicklei/go-restful/doc.go | 185 +++++++++ .../emicklei/go-restful/doc_examples_test.go | 41 ++ .../emicklei/go-restful/entity_accessors.go | 162 ++++++++ .../go-restful/entity_accessors_test.go | 69 ++++ .../emicklei/go-restful/examples/.goconvey | 1 + .../examples/google_app_engine/.goconvey | 1 + .../examples/google_app_engine/app.yaml | 20 + .../google_app_engine/datastore/.goconvey | 1 + .../google_app_engine/datastore/app.yaml | 18 + .../google_app_engine/datastore/main.go | 267 +++++++++++++ .../restful-appstats-integration.go | 12 + .../google_app_engine/restful-user-service.go | 162 ++++++++ .../emicklei/go-restful/examples/home.html | 7 + .../examples/msgpack/msgpack_entity.go | 34 ++ .../examples/msgpack/msgpack_entity_test.go | 160 ++++++++ .../examples/restful-CORS-filter.go | 68 ++++ .../examples/restful-NCSA-logging.go | 54 +++ .../examples/restful-basic-authentication.go | 35 ++ .../examples/restful-cpuprofiler-service.go | 65 +++ .../examples/restful-curly-router.go | 107 +++++ .../examples/restful-curly-router_test.go | 149 +++++++ .../examples/restful-encoding-filter.go | 61 +++ .../go-restful/examples/restful-filters.go | 114 ++++++ .../examples/restful-form-handling.go | 63 +++ .../examples/restful-hello-world.go | 23 ++ .../examples/restful-html-template.go | 35 ++ .../examples/restful-multi-containers.go | 43 ++ .../examples/restful-no-cache-filter.go | 25 ++ .../go-restful/examples/restful-openapi.go | 178 +++++++++ .../examples/restful-options-filter.go | 51 +++ .../go-restful/examples/restful-path-tail.go | 27 ++ .../examples/restful-pre-post-filters.go | 98 +++++ .../examples/restful-resource-functions.go | 63 +++ .../go-restful/examples/restful-route_test.go | 39 ++ .../examples/restful-routefunction_test.go | 29 ++ .../examples/restful-serve-static.go | 47 +++ .../go-restful/examples/restful-swagger.go | 61 +++ .../examples/restful-user-resource.go | 169 ++++++++ .../github.com/emicklei/go-restful/filter.go | 35 ++ .../emicklei/go-restful/filter_test.go | 141 +++++++ vendor/github.com/emicklei/go-restful/json.go | 11 + .../emicklei/go-restful/jsoniter.go | 12 + .../github.com/emicklei/go-restful/jsr311.go | 293 ++++++++++++++ .../emicklei/go-restful/jsr311_test.go | 332 ++++++++++++++++ .../github.com/emicklei/go-restful/log/log.go | 34 ++ .../github.com/emicklei/go-restful/logger.go | 32 ++ vendor/github.com/emicklei/go-restful/mime.go | 45 +++ .../emicklei/go-restful/mime_test.go | 17 + .../emicklei/go-restful/options_filter.go | 34 ++ .../go-restful/options_filter_test.go | 34 ++ .../emicklei/go-restful/parameter.go | 143 +++++++ .../emicklei/go-restful/path_expression.go | 74 ++++ .../go-restful/path_expression_test.go | 45 +++ .../emicklei/go-restful/path_processor.go | 63 +++ .../go-restful/path_processor_test.go | 55 +++ .../github.com/emicklei/go-restful/request.go | 118 ++++++ .../emicklei/go-restful/request_test.go | 155 ++++++++ .../emicklei/go-restful/response.go | 250 ++++++++++++ .../emicklei/go-restful/response_test.go | 213 ++++++++++ .../github.com/emicklei/go-restful/route.go | 149 +++++++ .../emicklei/go-restful/route_builder.go | 321 +++++++++++++++ .../emicklei/go-restful/route_builder_test.go | 76 ++++ .../emicklei/go-restful/route_test.go | 76 ++++ .../github.com/emicklei/go-restful/router.go | 20 + .../emicklei/go-restful/service_error.go | 23 ++ .../emicklei/go-restful/tracer_test.go | 18 + .../emicklei/go-restful/web_service.go | 290 ++++++++++++++ .../go-restful/web_service_container.go | 39 ++ .../emicklei/go-restful/web_service_test.go | 343 ++++++++++++++++ 95 files changed, 8561 insertions(+) create mode 100644 vendor/github.com/emicklei/go-restful/.gitignore create mode 100644 vendor/github.com/emicklei/go-restful/.travis.yml create mode 100644 vendor/github.com/emicklei/go-restful/CHANGES.md create mode 100644 vendor/github.com/emicklei/go-restful/LICENSE create mode 100644 vendor/github.com/emicklei/go-restful/Makefile create mode 100644 vendor/github.com/emicklei/go-restful/README.md create mode 100644 vendor/github.com/emicklei/go-restful/Srcfile create mode 100644 vendor/github.com/emicklei/go-restful/bench_curly_test.go create mode 100644 vendor/github.com/emicklei/go-restful/bench_test.go create mode 100644 vendor/github.com/emicklei/go-restful/bench_test.sh create mode 100644 vendor/github.com/emicklei/go-restful/compress.go create mode 100644 vendor/github.com/emicklei/go-restful/compress_test.go create mode 100644 vendor/github.com/emicklei/go-restful/compressor_cache.go create mode 100644 vendor/github.com/emicklei/go-restful/compressor_pools.go create mode 100644 vendor/github.com/emicklei/go-restful/compressors.go create mode 100644 vendor/github.com/emicklei/go-restful/constants.go create mode 100644 vendor/github.com/emicklei/go-restful/container.go create mode 100644 vendor/github.com/emicklei/go-restful/container_test.go create mode 100644 vendor/github.com/emicklei/go-restful/cors_filter.go create mode 100644 vendor/github.com/emicklei/go-restful/cors_filter_test.go create mode 100644 vendor/github.com/emicklei/go-restful/coverage.sh create mode 100644 vendor/github.com/emicklei/go-restful/curly.go create mode 100644 vendor/github.com/emicklei/go-restful/curly_route.go create mode 100644 vendor/github.com/emicklei/go-restful/curly_test.go create mode 100644 vendor/github.com/emicklei/go-restful/doc.go create mode 100644 vendor/github.com/emicklei/go-restful/doc_examples_test.go create mode 100644 vendor/github.com/emicklei/go-restful/entity_accessors.go create mode 100644 vendor/github.com/emicklei/go-restful/entity_accessors_test.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/.goconvey create mode 100644 vendor/github.com/emicklei/go-restful/examples/google_app_engine/.goconvey create mode 100644 vendor/github.com/emicklei/go-restful/examples/google_app_engine/app.yaml create mode 100644 vendor/github.com/emicklei/go-restful/examples/google_app_engine/datastore/.goconvey create mode 100644 vendor/github.com/emicklei/go-restful/examples/google_app_engine/datastore/app.yaml create mode 100644 vendor/github.com/emicklei/go-restful/examples/google_app_engine/datastore/main.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/google_app_engine/restful-appstats-integration.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/google_app_engine/restful-user-service.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/home.html create mode 100644 vendor/github.com/emicklei/go-restful/examples/msgpack/msgpack_entity.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/msgpack/msgpack_entity_test.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-CORS-filter.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-NCSA-logging.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-basic-authentication.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-cpuprofiler-service.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-curly-router.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-curly-router_test.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-encoding-filter.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-filters.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-form-handling.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-hello-world.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-html-template.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-multi-containers.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-no-cache-filter.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-openapi.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-options-filter.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-path-tail.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-pre-post-filters.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-resource-functions.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-route_test.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-routefunction_test.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-serve-static.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-swagger.go create mode 100644 vendor/github.com/emicklei/go-restful/examples/restful-user-resource.go create mode 100644 vendor/github.com/emicklei/go-restful/filter.go create mode 100644 vendor/github.com/emicklei/go-restful/filter_test.go create mode 100644 vendor/github.com/emicklei/go-restful/json.go create mode 100644 vendor/github.com/emicklei/go-restful/jsoniter.go create mode 100644 vendor/github.com/emicklei/go-restful/jsr311.go create mode 100644 vendor/github.com/emicklei/go-restful/jsr311_test.go create mode 100644 vendor/github.com/emicklei/go-restful/log/log.go create mode 100644 vendor/github.com/emicklei/go-restful/logger.go create mode 100644 vendor/github.com/emicklei/go-restful/mime.go create mode 100644 vendor/github.com/emicklei/go-restful/mime_test.go create mode 100644 vendor/github.com/emicklei/go-restful/options_filter.go create mode 100644 vendor/github.com/emicklei/go-restful/options_filter_test.go create mode 100644 vendor/github.com/emicklei/go-restful/parameter.go create mode 100644 vendor/github.com/emicklei/go-restful/path_expression.go create mode 100644 vendor/github.com/emicklei/go-restful/path_expression_test.go create mode 100644 vendor/github.com/emicklei/go-restful/path_processor.go create mode 100644 vendor/github.com/emicklei/go-restful/path_processor_test.go create mode 100644 vendor/github.com/emicklei/go-restful/request.go create mode 100644 vendor/github.com/emicklei/go-restful/request_test.go create mode 100644 vendor/github.com/emicklei/go-restful/response.go create mode 100644 vendor/github.com/emicklei/go-restful/response_test.go create mode 100644 vendor/github.com/emicklei/go-restful/route.go create mode 100644 vendor/github.com/emicklei/go-restful/route_builder.go create mode 100644 vendor/github.com/emicklei/go-restful/route_builder_test.go create mode 100644 vendor/github.com/emicklei/go-restful/route_test.go create mode 100644 vendor/github.com/emicklei/go-restful/router.go create mode 100644 vendor/github.com/emicklei/go-restful/service_error.go create mode 100644 vendor/github.com/emicklei/go-restful/tracer_test.go create mode 100644 vendor/github.com/emicklei/go-restful/web_service.go create mode 100644 vendor/github.com/emicklei/go-restful/web_service_container.go create mode 100644 vendor/github.com/emicklei/go-restful/web_service_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 8935fb0f3a1..386cb27a045 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -149,6 +149,17 @@ pruneopts = "" revision = "9f541cc9db5d55bce703bd99987c9d5cb8eea45e" +[[projects]] + digest = "1:11596f65b08621cab2bd8096c60e2b4f1d4c6bc1d44f4905ffcb067141f6938a" + name = "github.com/emicklei/go-restful" + packages = [ + ".", + "log", + ] + pruneopts = "" + revision = "3eb9738c1697594ea6e71a7156a9bb32ed216cf0" + version = "v2.8.0" + [[projects]] digest = "1:b13707423743d41665fd23f0c36b2f37bb49c30e94adb813319c44188a51ba22" name = "github.com/ghodss/yaml" @@ -1105,6 +1116,8 @@ packages = [ "pkg/features", "pkg/kubelet/apis", + "pkg/scheduler/api", + "pkg/scheduler/api/v1", ] pruneopts = "" revision = "81753b10df112992bf51bbc2c2f85208aad78335" @@ -1114,6 +1127,7 @@ analyzer-name = "dep" analyzer-version = 1 input-imports = [ + "github.com/emicklei/go-restful", "github.com/go-sql-driver/mysql", "github.com/golang/glog", "github.com/onsi/ginkgo", @@ -1165,6 +1179,7 @@ "k8s.io/code-generator/cmd/informer-gen", "k8s.io/code-generator/cmd/lister-gen", "k8s.io/kubernetes/pkg/kubelet/apis", + "k8s.io/kubernetes/pkg/scheduler/api/v1", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 7ae00105bf0..01c9a94f9b9 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -74,3 +74,7 @@ required = [ name = "github.com/pkg/errors" version = "0.9.0" source = "https://github.com/pingcap/errors.git" + +[[constraint]] + name = "github.com/emicklei/go-restful" + version = "v2.8.0" diff --git a/vendor/github.com/emicklei/go-restful/.gitignore b/vendor/github.com/emicklei/go-restful/.gitignore new file mode 100644 index 00000000000..cece7be6649 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/.gitignore @@ -0,0 +1,70 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe + +restful.html + +*.out + +tmp.prof + +go-restful.test + +examples/restful-basic-authentication + +examples/restful-encoding-filter + +examples/restful-filters + +examples/restful-hello-world + +examples/restful-resource-functions + +examples/restful-serve-static + +examples/restful-user-service + +*.DS_Store +examples/restful-user-resource + +examples/restful-multi-containers + +examples/restful-form-handling + +examples/restful-CORS-filter + +examples/restful-options-filter + +examples/restful-curly-router + +examples/restful-cpuprofiler-service + +examples/restful-pre-post-filters + +curly.prof + +examples/restful-NCSA-logging + +examples/restful-html-template + +s.html +restful-path-tail diff --git a/vendor/github.com/emicklei/go-restful/.travis.yml b/vendor/github.com/emicklei/go-restful/.travis.yml new file mode 100644 index 00000000000..b22f8f547ee --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/.travis.yml @@ -0,0 +1,6 @@ +language: go + +go: + - 1.x + +script: go test -v \ No newline at end of file diff --git a/vendor/github.com/emicklei/go-restful/CHANGES.md b/vendor/github.com/emicklei/go-restful/CHANGES.md new file mode 100644 index 00000000000..195449fd15e --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/CHANGES.md @@ -0,0 +1,242 @@ +Change history of go-restful += +v2.8.0 +- add Request.QueryParameters() +- add json-iterator (via build tag) +- disable vgo module (until log is moved) + +v2.7.1 +- add vgo module + +v2.6.1 +- add JSONNewDecoderFunc to allow custom JSON Decoder usage (go 1.10+) + +v2.6.0 +- Make JSR 311 routing and path param processing consistent +- Adding description to RouteBuilder.Reads() +- Update example for Swagger12 and OpenAPI + +2017-09-13 +- added route condition functions using `.If(func)` in route building. + +2017-02-16 +- solved issue #304, make operation names unique + +2017-01-30 + + [IMPORTANT] For swagger users, change your import statement to: + swagger "github.com/emicklei/go-restful-swagger12" + +- moved swagger 1.2 code to go-restful-swagger12 +- created TAG 2.0.0 + +2017-01-27 + +- remove defer request body close +- expose Dispatch for testing filters and Routefunctions +- swagger response model cannot be array +- created TAG 1.0.0 + +2016-12-22 + +- (API change) Remove code related to caching request content. Removes SetCacheReadEntity(doCache bool) + +2016-11-26 + +- Default change! now use CurlyRouter (was RouterJSR311) +- Default change! no more caching of request content +- Default change! do not recover from panics + +2016-09-22 + +- fix the DefaultRequestContentType feature + +2016-02-14 + +- take the qualify factor of the Accept header mediatype into account when deciding the contentype of the response +- add constructors for custom entity accessors for xml and json + +2015-09-27 + +- rename new WriteStatusAnd... to WriteHeaderAnd... for consistency + +2015-09-25 + +- fixed problem with changing Header after WriteHeader (issue 235) + +2015-09-14 + +- changed behavior of WriteHeader (immediate write) and WriteEntity (no status write) +- added support for custom EntityReaderWriters. + +2015-08-06 + +- add support for reading entities from compressed request content +- use sync.Pool for compressors of http response and request body +- add Description to Parameter for documentation in Swagger UI + +2015-03-20 + +- add configurable logging + +2015-03-18 + +- if not specified, the Operation is derived from the Route function + +2015-03-17 + +- expose Parameter creation functions +- make trace logger an interface +- fix OPTIONSFilter +- customize rendering of ServiceError +- JSR311 router now handles wildcards +- add Notes to Route + +2014-11-27 + +- (api add) PrettyPrint per response. (as proposed in #167) + +2014-11-12 + +- (api add) ApiVersion(.) for documentation in Swagger UI + +2014-11-10 + +- (api change) struct fields tagged with "description" show up in Swagger UI + +2014-10-31 + +- (api change) ReturnsError -> Returns +- (api add) RouteBuilder.Do(aBuilder) for DRY use of RouteBuilder +- fix swagger nested structs +- sort Swagger response messages by code + +2014-10-23 + +- (api add) ReturnsError allows you to document Http codes in swagger +- fixed problem with greedy CurlyRouter +- (api add) Access-Control-Max-Age in CORS +- add tracing functionality (injectable) for debugging purposes +- support JSON parse 64bit int +- fix empty parameters for swagger +- WebServicesUrl is now optional for swagger +- fixed duplicate AccessControlAllowOrigin in CORS +- (api change) expose ServeMux in container +- (api add) added AllowedDomains in CORS +- (api add) ParameterNamed for detailed documentation + +2014-04-16 + +- (api add) expose constructor of Request for testing. + +2014-06-27 + +- (api add) ParameterNamed gives access to a Parameter definition and its data (for further specification). +- (api add) SetCacheReadEntity allow scontrol over whether or not the request body is being cached (default true for compatibility reasons). + +2014-07-03 + +- (api add) CORS can be configured with a list of allowed domains + +2014-03-12 + +- (api add) Route path parameters can use wildcard or regular expressions. (requires CurlyRouter) + +2014-02-26 + +- (api add) Request now provides information about the matched Route, see method SelectedRoutePath + +2014-02-17 + +- (api change) renamed parameter constants (go-lint checks) + +2014-01-10 + +- (api add) support for CloseNotify, see http://golang.org/pkg/net/http/#CloseNotifier + +2014-01-07 + +- (api change) Write* methods in Response now return the error or nil. +- added example of serving HTML from a Go template. +- fixed comparing Allowed headers in CORS (is now case-insensitive) + +2013-11-13 + +- (api add) Response knows how many bytes are written to the response body. + +2013-10-29 + +- (api add) RecoverHandler(handler RecoverHandleFunction) to change how panic recovery is handled. Default behavior is to log and return a stacktrace. This may be a security issue as it exposes sourcecode information. + +2013-10-04 + +- (api add) Response knows what HTTP status has been written +- (api add) Request can have attributes (map of string->interface, also called request-scoped variables + +2013-09-12 + +- (api change) Router interface simplified +- Implemented CurlyRouter, a Router that does not use|allow regular expressions in paths + +2013-08-05 + - add OPTIONS support + - add CORS support + +2013-08-27 + +- fixed some reported issues (see github) +- (api change) deprecated use of WriteError; use WriteErrorString instead + +2014-04-15 + +- (fix) v1.0.1 tag: fix Issue 111: WriteErrorString + +2013-08-08 + +- (api add) Added implementation Container: a WebServices collection with its own http.ServeMux allowing multiple endpoints per program. Existing uses of go-restful will register their services to the DefaultContainer. +- (api add) the swagger package has be extended to have a UI per container. +- if panic is detected then a small stack trace is printed (thanks to runner-mei) +- (api add) WriteErrorString to Response + +Important API changes: + +- (api remove) package variable DoNotRecover no longer works ; use restful.DefaultContainer.DoNotRecover(true) instead. +- (api remove) package variable EnableContentEncoding no longer works ; use restful.DefaultContainer.EnableContentEncoding(true) instead. + + +2013-07-06 + +- (api add) Added support for response encoding (gzip and deflate(zlib)). This feature is disabled on default (for backwards compatibility). Use restful.EnableContentEncoding = true in your initialization to enable this feature. + +2013-06-19 + +- (improve) DoNotRecover option, moved request body closer, improved ReadEntity + +2013-06-03 + +- (api change) removed Dispatcher interface, hide PathExpression +- changed receiver names of type functions to be more idiomatic Go + +2013-06-02 + +- (optimize) Cache the RegExp compilation of Paths. + +2013-05-22 + +- (api add) Added support for request/response filter functions + +2013-05-18 + + +- (api add) Added feature to change the default Http Request Dispatch function (travis cline) +- (api change) Moved Swagger Webservice to swagger package (see example restful-user) + +[2012-11-14 .. 2013-05-18> + +- See https://github.com/emicklei/go-restful/commits + +2012-11-14 + +- Initial commit + + diff --git a/vendor/github.com/emicklei/go-restful/LICENSE b/vendor/github.com/emicklei/go-restful/LICENSE new file mode 100644 index 00000000000..ece7ec61eff --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2012,2013 Ernest Micklei + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/emicklei/go-restful/Makefile b/vendor/github.com/emicklei/go-restful/Makefile new file mode 100644 index 00000000000..b40081cc0e7 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/Makefile @@ -0,0 +1,7 @@ +all: test + +test: + go test -v . + +ex: + cd examples && ls *.go | xargs go build -o /tmp/ignore \ No newline at end of file diff --git a/vendor/github.com/emicklei/go-restful/README.md b/vendor/github.com/emicklei/go-restful/README.md new file mode 100644 index 00000000000..f52c25acf6e --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/README.md @@ -0,0 +1,88 @@ +go-restful +========== +package for building REST-style Web Services using Google Go + +[![Build Status](https://travis-ci.org/emicklei/go-restful.png)](https://travis-ci.org/emicklei/go-restful) +[![Go Report Card](https://goreportcard.com/badge/github.com/emicklei/go-restful)](https://goreportcard.com/report/github.com/emicklei/go-restful) +[![GoDoc](https://godoc.org/github.com/emicklei/go-restful?status.svg)](https://godoc.org/github.com/emicklei/go-restful) + +- [Code examples](https://github.com/emicklei/go-restful/tree/master/examples) + +REST asks developers to use HTTP methods explicitly and in a way that's consistent with the protocol definition. This basic REST design principle establishes a one-to-one mapping between create, read, update, and delete (CRUD) operations and HTTP methods. According to this mapping: + +- GET = Retrieve a representation of a resource +- POST = Create if you are sending content to the server to create a subordinate of the specified resource collection, using some server-side algorithm. +- PUT = Create if you are sending the full content of the specified resource (URI). +- PUT = Update if you are updating the full content of the specified resource. +- DELETE = Delete if you are requesting the server to delete the resource +- PATCH = Update partial content of a resource +- OPTIONS = Get information about the communication options for the request URI + +### Example + +```Go +ws := new(restful.WebService) +ws. + Path("/users"). + Consumes(restful.MIME_XML, restful.MIME_JSON). + Produces(restful.MIME_JSON, restful.MIME_XML) + +ws.Route(ws.GET("/{user-id}").To(u.findUser). + Doc("get a user"). + Param(ws.PathParameter("user-id", "identifier of the user").DataType("string")). + Writes(User{})) +... + +func (u UserResource) findUser(request *restful.Request, response *restful.Response) { + id := request.PathParameter("user-id") + ... +} +``` + +[Full API of a UserResource](https://github.com/emicklei/go-restful/tree/master/examples/restful-user-resource.go) + +### Features + +- Routes for request → function mapping with path parameter (e.g. {id}) support +- Configurable router: + - (default) Fast routing algorithm that allows static elements, regular expressions and dynamic parameters in the URL path (e.g. /meetings/{id} or /static/{subpath:*} + - Routing algorithm after [JSR311](http://jsr311.java.net/nonav/releases/1.1/spec/spec.html) that is implemented using (but does **not** accept) regular expressions +- Request API for reading structs from JSON/XML and accesing parameters (path,query,header) +- Response API for writing structs to JSON/XML and setting headers +- Customizable encoding using EntityReaderWriter registration +- Filters for intercepting the request → response flow on Service or Route level +- Request-scoped variables using attributes +- Containers for WebServices on different HTTP endpoints +- Content encoding (gzip,deflate) of request and response payloads +- Automatic responses on OPTIONS (using a filter) +- Automatic CORS request handling (using a filter) +- API declaration for Swagger UI ([go-restful-openapi](https://github.com/emicklei/go-restful-openapi), see [go-restful-swagger12](https://github.com/emicklei/go-restful-swagger12)) +- Panic recovery to produce HTTP 500, customizable using RecoverHandler(...) +- Route errors produce HTTP 404/405/406/415 errors, customizable using ServiceErrorHandler(...) +- Configurable (trace) logging +- Customizable gzip/deflate readers and writers using CompressorProvider registration + +## How to customize +There are several hooks to customize the behavior of the go-restful package. + +- Router algorithm +- Panic recovery +- JSON decoder +- Trace logging +- Compression +- Encoders for other serializers +- Use [jsoniter](https://github.com/json-iterator/go) by build this package using a tag, e.g. `go build -tags=jsoniter .` + +TODO: write examples of these. + +## Resources + +- [Example posted on blog](http://ernestmicklei.com/2012/11/go-restful-first-working-example/) +- [Design explained on blog](http://ernestmicklei.com/2012/11/go-restful-api-design/) +- [sourcegraph](https://sourcegraph.com/github.com/emicklei/go-restful) +- [showcase: Zazkia - tcp proxy for testing resiliency](https://github.com/emicklei/zazkia) +- [showcase: Mora - MongoDB REST Api server](https://github.com/emicklei/mora) + +Type ```git shortlog -s``` for a full list of contributors. + +© 2012 - 2018, http://ernestmicklei.com. MIT License. Contributions are welcome. diff --git a/vendor/github.com/emicklei/go-restful/Srcfile b/vendor/github.com/emicklei/go-restful/Srcfile new file mode 100644 index 00000000000..16fd186892e --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/Srcfile @@ -0,0 +1 @@ +{"SkipDirs": ["examples"]} diff --git a/vendor/github.com/emicklei/go-restful/bench_curly_test.go b/vendor/github.com/emicklei/go-restful/bench_curly_test.go new file mode 100644 index 00000000000..db6a1a75244 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/bench_curly_test.go @@ -0,0 +1,51 @@ +package restful + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" +) + +func setupCurly(container *Container) []string { + wsCount := 26 + rtCount := 26 + urisCurly := []string{} + + container.Router(CurlyRouter{}) + for i := 0; i < wsCount; i++ { + root := fmt.Sprintf("/%s/{%s}/", string(i+97), string(i+97)) + ws := new(WebService).Path(root) + for j := 0; j < rtCount; j++ { + sub := fmt.Sprintf("/%s2/{%s2}", string(j+97), string(j+97)) + ws.Route(ws.GET(sub).Consumes("application/xml").Produces("application/xml").To(echoCurly)) + } + container.Add(ws) + for _, each := range ws.Routes() { + urisCurly = append(urisCurly, "http://bench.com"+each.Path) + } + } + return urisCurly +} + +func echoCurly(req *Request, resp *Response) {} + +func BenchmarkManyCurly(b *testing.B) { + container := NewContainer() + urisCurly := setupCurly(container) + b.ResetTimer() + for t := 0; t < b.N; t++ { + for r := 0; r < 1000; r++ { + for _, each := range urisCurly { + sendNoReturnTo(each, container, t) + } + } + } +} + +func sendNoReturnTo(address string, container *Container, t int) { + httpRequest, _ := http.NewRequest("GET", address, nil) + httpRequest.Header.Set("Accept", "application/xml") + httpWriter := httptest.NewRecorder() + container.dispatch(httpWriter, httpRequest) +} diff --git a/vendor/github.com/emicklei/go-restful/bench_test.go b/vendor/github.com/emicklei/go-restful/bench_test.go new file mode 100644 index 00000000000..3e77c2d292b --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/bench_test.go @@ -0,0 +1,43 @@ +package restful + +import ( + "fmt" + "io" + "testing" +) + +var uris = []string{} + +func setup(container *Container) { + wsCount := 26 + rtCount := 26 + + for i := 0; i < wsCount; i++ { + root := fmt.Sprintf("/%s/{%s}/", string(i+97), string(i+97)) + ws := new(WebService).Path(root) + for j := 0; j < rtCount; j++ { + sub := fmt.Sprintf("/%s2/{%s2}", string(j+97), string(j+97)) + ws.Route(ws.GET(sub).To(echo)) + } + container.Add(ws) + for _, each := range ws.Routes() { + uris = append(uris, "http://bench.com"+each.Path) + } + } +} + +func echo(req *Request, resp *Response) { + io.WriteString(resp.ResponseWriter, "echo") +} + +func BenchmarkMany(b *testing.B) { + container := NewContainer() + setup(container) + b.ResetTimer() + for t := 0; t < b.N; t++ { + for _, each := range uris { + // println(each) + sendItTo(each, container) + } + } +} diff --git a/vendor/github.com/emicklei/go-restful/bench_test.sh b/vendor/github.com/emicklei/go-restful/bench_test.sh new file mode 100644 index 00000000000..47ffbe4ac9d --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/bench_test.sh @@ -0,0 +1,10 @@ +#go test -run=none -file bench_test.go -test.bench . -cpuprofile=bench_test.out + +go test -c +./go-restful.test -test.run=none -test.cpuprofile=tmp.prof -test.bench=BenchmarkMany +./go-restful.test -test.run=none -test.cpuprofile=curly.prof -test.bench=BenchmarkManyCurly + +#go tool pprof go-restful.test tmp.prof +go tool pprof go-restful.test curly.prof + + diff --git a/vendor/github.com/emicklei/go-restful/compress.go b/vendor/github.com/emicklei/go-restful/compress.go new file mode 100644 index 00000000000..220b37712f5 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/compress.go @@ -0,0 +1,123 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "bufio" + "compress/gzip" + "compress/zlib" + "errors" + "io" + "net" + "net/http" + "strings" +) + +// OBSOLETE : use restful.DefaultContainer.EnableContentEncoding(true) to change this setting. +var EnableContentEncoding = false + +// CompressingResponseWriter is a http.ResponseWriter that can perform content encoding (gzip and zlib) +type CompressingResponseWriter struct { + writer http.ResponseWriter + compressor io.WriteCloser + encoding string +} + +// Header is part of http.ResponseWriter interface +func (c *CompressingResponseWriter) Header() http.Header { + return c.writer.Header() +} + +// WriteHeader is part of http.ResponseWriter interface +func (c *CompressingResponseWriter) WriteHeader(status int) { + c.writer.WriteHeader(status) +} + +// Write is part of http.ResponseWriter interface +// It is passed through the compressor +func (c *CompressingResponseWriter) Write(bytes []byte) (int, error) { + if c.isCompressorClosed() { + return -1, errors.New("Compressing error: tried to write data using closed compressor") + } + return c.compressor.Write(bytes) +} + +// CloseNotify is part of http.CloseNotifier interface +func (c *CompressingResponseWriter) CloseNotify() <-chan bool { + return c.writer.(http.CloseNotifier).CloseNotify() +} + +// Close the underlying compressor +func (c *CompressingResponseWriter) Close() error { + if c.isCompressorClosed() { + return errors.New("Compressing error: tried to close already closed compressor") + } + + c.compressor.Close() + if ENCODING_GZIP == c.encoding { + currentCompressorProvider.ReleaseGzipWriter(c.compressor.(*gzip.Writer)) + } + if ENCODING_DEFLATE == c.encoding { + currentCompressorProvider.ReleaseZlibWriter(c.compressor.(*zlib.Writer)) + } + // gc hint needed? + c.compressor = nil + return nil +} + +func (c *CompressingResponseWriter) isCompressorClosed() bool { + return nil == c.compressor +} + +// Hijack implements the Hijacker interface +// This is especially useful when combining Container.EnabledContentEncoding +// in combination with websockets (for instance gorilla/websocket) +func (c *CompressingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hijacker, ok := c.writer.(http.Hijacker) + if !ok { + return nil, nil, errors.New("ResponseWriter doesn't support Hijacker interface") + } + return hijacker.Hijack() +} + +// WantsCompressedResponse reads the Accept-Encoding header to see if and which encoding is requested. +func wantsCompressedResponse(httpRequest *http.Request) (bool, string) { + header := httpRequest.Header.Get(HEADER_AcceptEncoding) + gi := strings.Index(header, ENCODING_GZIP) + zi := strings.Index(header, ENCODING_DEFLATE) + // use in order of appearance + if gi == -1 { + return zi != -1, ENCODING_DEFLATE + } else if zi == -1 { + return gi != -1, ENCODING_GZIP + } else { + if gi < zi { + return true, ENCODING_GZIP + } + return true, ENCODING_DEFLATE + } +} + +// NewCompressingResponseWriter create a CompressingResponseWriter for a known encoding = {gzip,deflate} +func NewCompressingResponseWriter(httpWriter http.ResponseWriter, encoding string) (*CompressingResponseWriter, error) { + httpWriter.Header().Set(HEADER_ContentEncoding, encoding) + c := new(CompressingResponseWriter) + c.writer = httpWriter + var err error + if ENCODING_GZIP == encoding { + w := currentCompressorProvider.AcquireGzipWriter() + w.Reset(httpWriter) + c.compressor = w + c.encoding = ENCODING_GZIP + } else if ENCODING_DEFLATE == encoding { + w := currentCompressorProvider.AcquireZlibWriter() + w.Reset(httpWriter) + c.compressor = w + c.encoding = ENCODING_DEFLATE + } else { + return nil, errors.New("Unknown encoding:" + encoding) + } + return c, err +} diff --git a/vendor/github.com/emicklei/go-restful/compress_test.go b/vendor/github.com/emicklei/go-restful/compress_test.go new file mode 100644 index 00000000000..cc3e93d5431 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/compress_test.go @@ -0,0 +1,125 @@ +package restful + +import ( + "bytes" + "compress/gzip" + "compress/zlib" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" +) + +// go test -v -test.run TestGzip ...restful +func TestGzip(t *testing.T) { + EnableContentEncoding = true + httpRequest, _ := http.NewRequest("GET", "/test", nil) + httpRequest.Header.Set("Accept-Encoding", "gzip,deflate") + httpWriter := httptest.NewRecorder() + wanted, encoding := wantsCompressedResponse(httpRequest) + if !wanted { + t.Fatal("should accept gzip") + } + if encoding != "gzip" { + t.Fatal("expected gzip") + } + c, err := NewCompressingResponseWriter(httpWriter, encoding) + if err != nil { + t.Fatal(err.Error()) + } + c.Write([]byte("Hello World")) + c.Close() + if httpWriter.Header().Get("Content-Encoding") != "gzip" { + t.Fatal("Missing gzip header") + } + reader, err := gzip.NewReader(httpWriter.Body) + if err != nil { + t.Fatal(err.Error()) + } + data, err := ioutil.ReadAll(reader) + if err != nil { + t.Fatal(err.Error()) + } + if got, want := string(data), "Hello World"; got != want { + t.Errorf("got %v want %v", got, want) + } +} + +func TestDeflate(t *testing.T) { + EnableContentEncoding = true + httpRequest, _ := http.NewRequest("GET", "/test", nil) + httpRequest.Header.Set("Accept-Encoding", "deflate,gzip") + httpWriter := httptest.NewRecorder() + wanted, encoding := wantsCompressedResponse(httpRequest) + if !wanted { + t.Fatal("should accept deflate") + } + if encoding != "deflate" { + t.Fatal("expected deflate") + } + c, err := NewCompressingResponseWriter(httpWriter, encoding) + if err != nil { + t.Fatal(err.Error()) + } + c.Write([]byte("Hello World")) + c.Close() + if httpWriter.Header().Get("Content-Encoding") != "deflate" { + t.Fatal("Missing deflate header") + } + reader, err := zlib.NewReader(httpWriter.Body) + if err != nil { + t.Fatal(err.Error()) + } + data, err := ioutil.ReadAll(reader) + if err != nil { + t.Fatal(err.Error()) + } + if got, want := string(data), "Hello World"; got != want { + t.Errorf("got %v want %v", got, want) + } +} + +func TestGzipDecompressRequestBody(t *testing.T) { + b := new(bytes.Buffer) + w := newGzipWriter() + w.Reset(b) + io.WriteString(w, `{"msg":"hi"}`) + w.Flush() + w.Close() + + req := new(Request) + httpRequest, _ := http.NewRequest("GET", "/", bytes.NewReader(b.Bytes())) + httpRequest.Header.Set("Content-Type", "application/json") + httpRequest.Header.Set("Content-Encoding", "gzip") + req.Request = httpRequest + + doc := make(map[string]interface{}) + req.ReadEntity(&doc) + + if got, want := doc["msg"], "hi"; got != want { + t.Errorf("got %v want %v", got, want) + } +} + +func TestZlibDecompressRequestBody(t *testing.T) { + b := new(bytes.Buffer) + w := newZlibWriter() + w.Reset(b) + io.WriteString(w, `{"msg":"hi"}`) + w.Flush() + w.Close() + + req := new(Request) + httpRequest, _ := http.NewRequest("GET", "/", bytes.NewReader(b.Bytes())) + httpRequest.Header.Set("Content-Type", "application/json") + httpRequest.Header.Set("Content-Encoding", "deflate") + req.Request = httpRequest + + doc := make(map[string]interface{}) + req.ReadEntity(&doc) + + if got, want := doc["msg"], "hi"; got != want { + t.Errorf("got %v want %v", got, want) + } +} diff --git a/vendor/github.com/emicklei/go-restful/compressor_cache.go b/vendor/github.com/emicklei/go-restful/compressor_cache.go new file mode 100644 index 00000000000..ee426010a2d --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/compressor_cache.go @@ -0,0 +1,103 @@ +package restful + +// Copyright 2015 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "compress/gzip" + "compress/zlib" +) + +// BoundedCachedCompressors is a CompressorProvider that uses a cache with a fixed amount +// of writers and readers (resources). +// If a new resource is acquired and all are in use, it will return a new unmanaged resource. +type BoundedCachedCompressors struct { + gzipWriters chan *gzip.Writer + gzipReaders chan *gzip.Reader + zlibWriters chan *zlib.Writer + writersCapacity int + readersCapacity int +} + +// NewBoundedCachedCompressors returns a new, with filled cache, BoundedCachedCompressors. +func NewBoundedCachedCompressors(writersCapacity, readersCapacity int) *BoundedCachedCompressors { + b := &BoundedCachedCompressors{ + gzipWriters: make(chan *gzip.Writer, writersCapacity), + gzipReaders: make(chan *gzip.Reader, readersCapacity), + zlibWriters: make(chan *zlib.Writer, writersCapacity), + writersCapacity: writersCapacity, + readersCapacity: readersCapacity, + } + for ix := 0; ix < writersCapacity; ix++ { + b.gzipWriters <- newGzipWriter() + b.zlibWriters <- newZlibWriter() + } + for ix := 0; ix < readersCapacity; ix++ { + b.gzipReaders <- newGzipReader() + } + return b +} + +// AcquireGzipWriter returns an resettable *gzip.Writer. Needs to be released. +func (b *BoundedCachedCompressors) AcquireGzipWriter() *gzip.Writer { + var writer *gzip.Writer + select { + case writer, _ = <-b.gzipWriters: + default: + // return a new unmanaged one + writer = newGzipWriter() + } + return writer +} + +// ReleaseGzipWriter accepts a writer (does not have to be one that was cached) +// only when the cache has room for it. It will ignore it otherwise. +func (b *BoundedCachedCompressors) ReleaseGzipWriter(w *gzip.Writer) { + // forget the unmanaged ones + if len(b.gzipWriters) < b.writersCapacity { + b.gzipWriters <- w + } +} + +// AcquireGzipReader returns a *gzip.Reader. Needs to be released. +func (b *BoundedCachedCompressors) AcquireGzipReader() *gzip.Reader { + var reader *gzip.Reader + select { + case reader, _ = <-b.gzipReaders: + default: + // return a new unmanaged one + reader = newGzipReader() + } + return reader +} + +// ReleaseGzipReader accepts a reader (does not have to be one that was cached) +// only when the cache has room for it. It will ignore it otherwise. +func (b *BoundedCachedCompressors) ReleaseGzipReader(r *gzip.Reader) { + // forget the unmanaged ones + if len(b.gzipReaders) < b.readersCapacity { + b.gzipReaders <- r + } +} + +// AcquireZlibWriter returns an resettable *zlib.Writer. Needs to be released. +func (b *BoundedCachedCompressors) AcquireZlibWriter() *zlib.Writer { + var writer *zlib.Writer + select { + case writer, _ = <-b.zlibWriters: + default: + // return a new unmanaged one + writer = newZlibWriter() + } + return writer +} + +// ReleaseZlibWriter accepts a writer (does not have to be one that was cached) +// only when the cache has room for it. It will ignore it otherwise. +func (b *BoundedCachedCompressors) ReleaseZlibWriter(w *zlib.Writer) { + // forget the unmanaged ones + if len(b.zlibWriters) < b.writersCapacity { + b.zlibWriters <- w + } +} diff --git a/vendor/github.com/emicklei/go-restful/compressor_pools.go b/vendor/github.com/emicklei/go-restful/compressor_pools.go new file mode 100644 index 00000000000..d866ce64bba --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/compressor_pools.go @@ -0,0 +1,91 @@ +package restful + +// Copyright 2015 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "bytes" + "compress/gzip" + "compress/zlib" + "sync" +) + +// SyncPoolCompessors is a CompressorProvider that use the standard sync.Pool. +type SyncPoolCompessors struct { + GzipWriterPool *sync.Pool + GzipReaderPool *sync.Pool + ZlibWriterPool *sync.Pool +} + +// NewSyncPoolCompessors returns a new ("empty") SyncPoolCompessors. +func NewSyncPoolCompessors() *SyncPoolCompessors { + return &SyncPoolCompessors{ + GzipWriterPool: &sync.Pool{ + New: func() interface{} { return newGzipWriter() }, + }, + GzipReaderPool: &sync.Pool{ + New: func() interface{} { return newGzipReader() }, + }, + ZlibWriterPool: &sync.Pool{ + New: func() interface{} { return newZlibWriter() }, + }, + } +} + +func (s *SyncPoolCompessors) AcquireGzipWriter() *gzip.Writer { + return s.GzipWriterPool.Get().(*gzip.Writer) +} + +func (s *SyncPoolCompessors) ReleaseGzipWriter(w *gzip.Writer) { + s.GzipWriterPool.Put(w) +} + +func (s *SyncPoolCompessors) AcquireGzipReader() *gzip.Reader { + return s.GzipReaderPool.Get().(*gzip.Reader) +} + +func (s *SyncPoolCompessors) ReleaseGzipReader(r *gzip.Reader) { + s.GzipReaderPool.Put(r) +} + +func (s *SyncPoolCompessors) AcquireZlibWriter() *zlib.Writer { + return s.ZlibWriterPool.Get().(*zlib.Writer) +} + +func (s *SyncPoolCompessors) ReleaseZlibWriter(w *zlib.Writer) { + s.ZlibWriterPool.Put(w) +} + +func newGzipWriter() *gzip.Writer { + // create with an empty bytes writer; it will be replaced before using the gzipWriter + writer, err := gzip.NewWriterLevel(new(bytes.Buffer), gzip.BestSpeed) + if err != nil { + panic(err.Error()) + } + return writer +} + +func newGzipReader() *gzip.Reader { + // create with an empty reader (but with GZIP header); it will be replaced before using the gzipReader + // we can safely use currentCompressProvider because it is set on package initialization. + w := currentCompressorProvider.AcquireGzipWriter() + defer currentCompressorProvider.ReleaseGzipWriter(w) + b := new(bytes.Buffer) + w.Reset(b) + w.Flush() + w.Close() + reader, err := gzip.NewReader(bytes.NewReader(b.Bytes())) + if err != nil { + panic(err.Error()) + } + return reader +} + +func newZlibWriter() *zlib.Writer { + writer, err := zlib.NewWriterLevel(new(bytes.Buffer), gzip.BestSpeed) + if err != nil { + panic(err.Error()) + } + return writer +} diff --git a/vendor/github.com/emicklei/go-restful/compressors.go b/vendor/github.com/emicklei/go-restful/compressors.go new file mode 100644 index 00000000000..9db4a8c8e97 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/compressors.go @@ -0,0 +1,54 @@ +package restful + +// Copyright 2015 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "compress/gzip" + "compress/zlib" +) + +// CompressorProvider describes a component that can provider compressors for the std methods. +type CompressorProvider interface { + // Returns a *gzip.Writer which needs to be released later. + // Before using it, call Reset(). + AcquireGzipWriter() *gzip.Writer + + // Releases an acquired *gzip.Writer. + ReleaseGzipWriter(w *gzip.Writer) + + // Returns a *gzip.Reader which needs to be released later. + AcquireGzipReader() *gzip.Reader + + // Releases an acquired *gzip.Reader. + ReleaseGzipReader(w *gzip.Reader) + + // Returns a *zlib.Writer which needs to be released later. + // Before using it, call Reset(). + AcquireZlibWriter() *zlib.Writer + + // Releases an acquired *zlib.Writer. + ReleaseZlibWriter(w *zlib.Writer) +} + +// DefaultCompressorProvider is the actual provider of compressors (zlib or gzip). +var currentCompressorProvider CompressorProvider + +func init() { + currentCompressorProvider = NewSyncPoolCompessors() +} + +// CurrentCompressorProvider returns the current CompressorProvider. +// It is initialized using a SyncPoolCompessors. +func CurrentCompressorProvider() CompressorProvider { + return currentCompressorProvider +} + +// SetCompressorProvider sets the actual provider of compressors (zlib or gzip). +func SetCompressorProvider(p CompressorProvider) { + if p == nil { + panic("cannot set compressor provider to nil") + } + currentCompressorProvider = p +} diff --git a/vendor/github.com/emicklei/go-restful/constants.go b/vendor/github.com/emicklei/go-restful/constants.go new file mode 100644 index 00000000000..203439c5e5f --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/constants.go @@ -0,0 +1,30 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +const ( + MIME_XML = "application/xml" // Accept or Content-Type used in Consumes() and/or Produces() + MIME_JSON = "application/json" // Accept or Content-Type used in Consumes() and/or Produces() + MIME_OCTET = "application/octet-stream" // If Content-Type is not present in request, use the default + + HEADER_Allow = "Allow" + HEADER_Accept = "Accept" + HEADER_Origin = "Origin" + HEADER_ContentType = "Content-Type" + HEADER_LastModified = "Last-Modified" + HEADER_AcceptEncoding = "Accept-Encoding" + HEADER_ContentEncoding = "Content-Encoding" + HEADER_AccessControlExposeHeaders = "Access-Control-Expose-Headers" + HEADER_AccessControlRequestMethod = "Access-Control-Request-Method" + HEADER_AccessControlRequestHeaders = "Access-Control-Request-Headers" + HEADER_AccessControlAllowMethods = "Access-Control-Allow-Methods" + HEADER_AccessControlAllowOrigin = "Access-Control-Allow-Origin" + HEADER_AccessControlAllowCredentials = "Access-Control-Allow-Credentials" + HEADER_AccessControlAllowHeaders = "Access-Control-Allow-Headers" + HEADER_AccessControlMaxAge = "Access-Control-Max-Age" + + ENCODING_GZIP = "gzip" + ENCODING_DEFLATE = "deflate" +) diff --git a/vendor/github.com/emicklei/go-restful/container.go b/vendor/github.com/emicklei/go-restful/container.go new file mode 100644 index 00000000000..b4ad153e8db --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/container.go @@ -0,0 +1,371 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "bytes" + "errors" + "fmt" + "net/http" + "os" + "runtime" + "strings" + "sync" + + "github.com/emicklei/go-restful/log" +) + +// Container holds a collection of WebServices and a http.ServeMux to dispatch http requests. +// The requests are further dispatched to routes of WebServices using a RouteSelector +type Container struct { + webServicesLock sync.RWMutex + webServices []*WebService + ServeMux *http.ServeMux + isRegisteredOnRoot bool + containerFilters []FilterFunction + doNotRecover bool // default is true + recoverHandleFunc RecoverHandleFunction + serviceErrorHandleFunc ServiceErrorHandleFunction + router RouteSelector // default is a CurlyRouter (RouterJSR311 is a slower alternative) + contentEncodingEnabled bool // default is false +} + +// NewContainer creates a new Container using a new ServeMux and default router (CurlyRouter) +func NewContainer() *Container { + return &Container{ + webServices: []*WebService{}, + ServeMux: http.NewServeMux(), + isRegisteredOnRoot: false, + containerFilters: []FilterFunction{}, + doNotRecover: true, + recoverHandleFunc: logStackOnRecover, + serviceErrorHandleFunc: writeServiceError, + router: CurlyRouter{}, + contentEncodingEnabled: false} +} + +// RecoverHandleFunction declares functions that can be used to handle a panic situation. +// The first argument is what recover() returns. The second must be used to communicate an error response. +type RecoverHandleFunction func(interface{}, http.ResponseWriter) + +// RecoverHandler changes the default function (logStackOnRecover) to be called +// when a panic is detected. DoNotRecover must be have its default value (=false). +func (c *Container) RecoverHandler(handler RecoverHandleFunction) { + c.recoverHandleFunc = handler +} + +// ServiceErrorHandleFunction declares functions that can be used to handle a service error situation. +// The first argument is the service error, the second is the request that resulted in the error and +// the third must be used to communicate an error response. +type ServiceErrorHandleFunction func(ServiceError, *Request, *Response) + +// ServiceErrorHandler changes the default function (writeServiceError) to be called +// when a ServiceError is detected. +func (c *Container) ServiceErrorHandler(handler ServiceErrorHandleFunction) { + c.serviceErrorHandleFunc = handler +} + +// DoNotRecover controls whether panics will be caught to return HTTP 500. +// If set to true, Route functions are responsible for handling any error situation. +// Default value is true. +func (c *Container) DoNotRecover(doNot bool) { + c.doNotRecover = doNot +} + +// Router changes the default Router (currently CurlyRouter) +func (c *Container) Router(aRouter RouteSelector) { + c.router = aRouter +} + +// EnableContentEncoding (default=false) allows for GZIP or DEFLATE encoding of responses. +func (c *Container) EnableContentEncoding(enabled bool) { + c.contentEncodingEnabled = enabled +} + +// Add a WebService to the Container. It will detect duplicate root paths and exit in that case. +func (c *Container) Add(service *WebService) *Container { + c.webServicesLock.Lock() + defer c.webServicesLock.Unlock() + + // if rootPath was not set then lazy initialize it + if len(service.rootPath) == 0 { + service.Path("/") + } + + // cannot have duplicate root paths + for _, each := range c.webServices { + if each.RootPath() == service.RootPath() { + log.Printf("[restful] WebService with duplicate root path detected:['%v']", each) + os.Exit(1) + } + } + + // If not registered on root then add specific mapping + if !c.isRegisteredOnRoot { + c.isRegisteredOnRoot = c.addHandler(service, c.ServeMux) + } + c.webServices = append(c.webServices, service) + return c +} + +// addHandler may set a new HandleFunc for the serveMux +// this function must run inside the critical region protected by the webServicesLock. +// returns true if the function was registered on root ("/") +func (c *Container) addHandler(service *WebService, serveMux *http.ServeMux) bool { + pattern := fixedPrefixPath(service.RootPath()) + // check if root path registration is needed + if "/" == pattern || "" == pattern { + serveMux.HandleFunc("/", c.dispatch) + return true + } + // detect if registration already exists + alreadyMapped := false + for _, each := range c.webServices { + if each.RootPath() == service.RootPath() { + alreadyMapped = true + break + } + } + if !alreadyMapped { + serveMux.HandleFunc(pattern, c.dispatch) + if !strings.HasSuffix(pattern, "/") { + serveMux.HandleFunc(pattern+"/", c.dispatch) + } + } + return false +} + +func (c *Container) Remove(ws *WebService) error { + if c.ServeMux == http.DefaultServeMux { + errMsg := fmt.Sprintf("[restful] cannot remove a WebService from a Container using the DefaultServeMux: ['%v']", ws) + log.Print(errMsg) + return errors.New(errMsg) + } + c.webServicesLock.Lock() + defer c.webServicesLock.Unlock() + // build a new ServeMux and re-register all WebServices + newServeMux := http.NewServeMux() + newServices := []*WebService{} + newIsRegisteredOnRoot := false + for _, each := range c.webServices { + if each.rootPath != ws.rootPath { + // If not registered on root then add specific mapping + if !newIsRegisteredOnRoot { + newIsRegisteredOnRoot = c.addHandler(each, newServeMux) + } + newServices = append(newServices, each) + } + } + c.webServices, c.ServeMux, c.isRegisteredOnRoot = newServices, newServeMux, newIsRegisteredOnRoot + return nil +} + +// logStackOnRecover is the default RecoverHandleFunction and is called +// when DoNotRecover is false and the recoverHandleFunc is not set for the container. +// Default implementation logs the stacktrace and writes the stacktrace on the response. +// This may be a security issue as it exposes sourcecode information. +func logStackOnRecover(panicReason interface{}, httpWriter http.ResponseWriter) { + var buffer bytes.Buffer + buffer.WriteString(fmt.Sprintf("[restful] recover from panic situation: - %v\r\n", panicReason)) + for i := 2; ; i += 1 { + _, file, line, ok := runtime.Caller(i) + if !ok { + break + } + buffer.WriteString(fmt.Sprintf(" %s:%d\r\n", file, line)) + } + log.Print(buffer.String()) + httpWriter.WriteHeader(http.StatusInternalServerError) + httpWriter.Write(buffer.Bytes()) +} + +// writeServiceError is the default ServiceErrorHandleFunction and is called +// when a ServiceError is returned during route selection. Default implementation +// calls resp.WriteErrorString(err.Code, err.Message) +func writeServiceError(err ServiceError, req *Request, resp *Response) { + resp.WriteErrorString(err.Code, err.Message) +} + +// Dispatch the incoming Http Request to a matching WebService. +func (c *Container) Dispatch(httpWriter http.ResponseWriter, httpRequest *http.Request) { + if httpWriter == nil { + panic("httpWriter cannot be nil") + } + if httpRequest == nil { + panic("httpRequest cannot be nil") + } + c.dispatch(httpWriter, httpRequest) +} + +// Dispatch the incoming Http Request to a matching WebService. +func (c *Container) dispatch(httpWriter http.ResponseWriter, httpRequest *http.Request) { + writer := httpWriter + + // CompressingResponseWriter should be closed after all operations are done + defer func() { + if compressWriter, ok := writer.(*CompressingResponseWriter); ok { + compressWriter.Close() + } + }() + + // Instal panic recovery unless told otherwise + if !c.doNotRecover { // catch all for 500 response + defer func() { + if r := recover(); r != nil { + c.recoverHandleFunc(r, writer) + return + } + }() + } + + // Detect if compression is needed + // assume without compression, test for override + if c.contentEncodingEnabled { + doCompress, encoding := wantsCompressedResponse(httpRequest) + if doCompress { + var err error + writer, err = NewCompressingResponseWriter(httpWriter, encoding) + if err != nil { + log.Print("[restful] unable to install compressor: ", err) + httpWriter.WriteHeader(http.StatusInternalServerError) + return + } + } + } + // Find best match Route ; err is non nil if no match was found + var webService *WebService + var route *Route + var err error + func() { + c.webServicesLock.RLock() + defer c.webServicesLock.RUnlock() + webService, route, err = c.router.SelectRoute( + c.webServices, + httpRequest) + }() + if err != nil { + // a non-200 response has already been written + // run container filters anyway ; they should not touch the response... + chain := FilterChain{Filters: c.containerFilters, Target: func(req *Request, resp *Response) { + switch err.(type) { + case ServiceError: + ser := err.(ServiceError) + c.serviceErrorHandleFunc(ser, req, resp) + } + // TODO + }} + chain.ProcessFilter(NewRequest(httpRequest), NewResponse(writer)) + return + } + pathProcessor, routerProcessesPath := c.router.(PathProcessor) + if !routerProcessesPath { + pathProcessor = defaultPathProcessor{} + } + pathParams := pathProcessor.ExtractParameters(route, webService, httpRequest.URL.Path) + wrappedRequest, wrappedResponse := route.wrapRequestResponse(writer, httpRequest, pathParams) + // pass through filters (if any) + if len(c.containerFilters)+len(webService.filters)+len(route.Filters) > 0 { + // compose filter chain + allFilters := []FilterFunction{} + allFilters = append(allFilters, c.containerFilters...) + allFilters = append(allFilters, webService.filters...) + allFilters = append(allFilters, route.Filters...) + chain := FilterChain{Filters: allFilters, Target: func(req *Request, resp *Response) { + // handle request by route after passing all filters + route.Function(wrappedRequest, wrappedResponse) + }} + chain.ProcessFilter(wrappedRequest, wrappedResponse) + } else { + // no filters, handle request by route + route.Function(wrappedRequest, wrappedResponse) + } +} + +// fixedPrefixPath returns the fixed part of the partspec ; it may include template vars {} +func fixedPrefixPath(pathspec string) string { + varBegin := strings.Index(pathspec, "{") + if -1 == varBegin { + return pathspec + } + return pathspec[:varBegin] +} + +// ServeHTTP implements net/http.Handler therefore a Container can be a Handler in a http.Server +func (c *Container) ServeHTTP(httpwriter http.ResponseWriter, httpRequest *http.Request) { + c.ServeMux.ServeHTTP(httpwriter, httpRequest) +} + +// Handle registers the handler for the given pattern. If a handler already exists for pattern, Handle panics. +func (c *Container) Handle(pattern string, handler http.Handler) { + c.ServeMux.Handle(pattern, handler) +} + +// HandleWithFilter registers the handler for the given pattern. +// Container's filter chain is applied for handler. +// If a handler already exists for pattern, HandleWithFilter panics. +func (c *Container) HandleWithFilter(pattern string, handler http.Handler) { + f := func(httpResponse http.ResponseWriter, httpRequest *http.Request) { + if len(c.containerFilters) == 0 { + handler.ServeHTTP(httpResponse, httpRequest) + return + } + + chain := FilterChain{Filters: c.containerFilters, Target: func(req *Request, resp *Response) { + handler.ServeHTTP(httpResponse, httpRequest) + }} + chain.ProcessFilter(NewRequest(httpRequest), NewResponse(httpResponse)) + } + + c.Handle(pattern, http.HandlerFunc(f)) +} + +// Filter appends a container FilterFunction. These are called before dispatching +// a http.Request to a WebService from the container +func (c *Container) Filter(filter FilterFunction) { + c.containerFilters = append(c.containerFilters, filter) +} + +// RegisteredWebServices returns the collections of added WebServices +func (c *Container) RegisteredWebServices() []*WebService { + c.webServicesLock.RLock() + defer c.webServicesLock.RUnlock() + result := make([]*WebService, len(c.webServices)) + for ix := range c.webServices { + result[ix] = c.webServices[ix] + } + return result +} + +// computeAllowedMethods returns a list of HTTP methods that are valid for a Request +func (c *Container) computeAllowedMethods(req *Request) []string { + // Go through all RegisteredWebServices() and all its Routes to collect the options + methods := []string{} + requestPath := req.Request.URL.Path + for _, ws := range c.RegisteredWebServices() { + matches := ws.pathExpr.Matcher.FindStringSubmatch(requestPath) + if matches != nil { + finalMatch := matches[len(matches)-1] + for _, rt := range ws.Routes() { + matches := rt.pathExpr.Matcher.FindStringSubmatch(finalMatch) + if matches != nil { + lastMatch := matches[len(matches)-1] + if lastMatch == "" || lastMatch == "/" { // do not include if value is neither empty nor ‘/’. + methods = append(methods, rt.Method) + } + } + } + } + } + // methods = append(methods, "OPTIONS") not sure about this + return methods +} + +// newBasicRequestResponse creates a pair of Request,Response from its http versions. +// It is basic because no parameter or (produces) content-type information is given. +func newBasicRequestResponse(httpWriter http.ResponseWriter, httpRequest *http.Request) (*Request, *Response) { + resp := NewResponse(httpWriter) + resp.requestAccept = httpRequest.Header.Get(HEADER_Accept) + return NewRequest(httpRequest), resp +} diff --git a/vendor/github.com/emicklei/go-restful/container_test.go b/vendor/github.com/emicklei/go-restful/container_test.go new file mode 100644 index 00000000000..491c793ab31 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/container_test.go @@ -0,0 +1,83 @@ +package restful + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +// go test -v -test.run TestContainer_computeAllowedMethods ...restful +func TestContainer_computeAllowedMethods(t *testing.T) { + wc := NewContainer() + ws1 := new(WebService).Path("/users") + ws1.Route(ws1.GET("{i}").To(dummy)) + ws1.Route(ws1.POST("{i}").To(dummy)) + wc.Add(ws1) + httpRequest, _ := http.NewRequest("GET", "http://api.his.com/users/1", nil) + rreq := Request{Request: httpRequest} + m := wc.computeAllowedMethods(&rreq) + if len(m) != 2 { + t.Errorf("got %d expected 2 methods, %v", len(m), m) + } +} + +func TestContainer_HandleWithFilter(t *testing.T) { + prefilterCalled := false + postfilterCalled := false + httpHandlerCalled := false + + wc := NewContainer() + wc.Filter(func(request *Request, response *Response, chain *FilterChain) { + prefilterCalled = true + chain.ProcessFilter(request, response) + }) + wc.HandleWithFilter("/", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + httpHandlerCalled = true + w.Write([]byte("ok")) + })) + wc.Filter(func(request *Request, response *Response, chain *FilterChain) { + postfilterCalled = true + chain.ProcessFilter(request, response) + }) + + recorder := httptest.NewRecorder() + request, _ := http.NewRequest("GET", "/", nil) + wc.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Errorf("unexpected code %d", recorder.Code) + } + if recorder.Body.String() != "ok" { + t.Errorf("unexpected body %s", recorder.Body.String()) + } + if !prefilterCalled { + t.Errorf("filter added before calling HandleWithFilter wasn't called") + } + if !postfilterCalled { + t.Errorf("filter added after calling HandleWithFilter wasn't called") + } + if !httpHandlerCalled { + t.Errorf("handler added by calling HandleWithFilter wasn't called") + } +} + +func TestContainerAddAndRemove(t *testing.T) { + ws1 := new(WebService).Path("/") + ws2 := new(WebService).Path("/users") + wc := NewContainer() + wc.Add(ws1) + wc.Add(ws2) + wc.Remove(ws2) + if len(wc.webServices) != 1 { + t.Errorf("expected one webservices") + } + if !wc.isRegisteredOnRoot { + t.Errorf("expected on root registered") + } + wc.Remove(ws1) + if len(wc.webServices) > 0 { + t.Errorf("expected zero webservices") + } + if wc.isRegisteredOnRoot { + t.Errorf("expected not on root registered") + } +} diff --git a/vendor/github.com/emicklei/go-restful/cors_filter.go b/vendor/github.com/emicklei/go-restful/cors_filter.go new file mode 100644 index 00000000000..1efeef072d0 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/cors_filter.go @@ -0,0 +1,202 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "regexp" + "strconv" + "strings" +) + +// CrossOriginResourceSharing is used to create a Container Filter that implements CORS. +// Cross-origin resource sharing (CORS) is a mechanism that allows JavaScript on a web page +// to make XMLHttpRequests to another domain, not the domain the JavaScript originated from. +// +// http://en.wikipedia.org/wiki/Cross-origin_resource_sharing +// http://enable-cors.org/server.html +// http://www.html5rocks.com/en/tutorials/cors/#toc-handling-a-not-so-simple-request +type CrossOriginResourceSharing struct { + ExposeHeaders []string // list of Header names + AllowedHeaders []string // list of Header names + AllowedDomains []string // list of allowed values for Http Origin. An allowed value can be a regular expression to support subdomain matching. If empty all are allowed. + AllowedMethods []string + MaxAge int // number of seconds before requiring new Options request + CookiesAllowed bool + Container *Container + + allowedOriginPatterns []*regexp.Regexp // internal field for origin regexp check. +} + +// Filter is a filter function that implements the CORS flow as documented on http://enable-cors.org/server.html +// and http://www.html5rocks.com/static/images/cors_server_flowchart.png +func (c CrossOriginResourceSharing) Filter(req *Request, resp *Response, chain *FilterChain) { + origin := req.Request.Header.Get(HEADER_Origin) + if len(origin) == 0 { + if trace { + traceLogger.Print("no Http header Origin set") + } + chain.ProcessFilter(req, resp) + return + } + if !c.isOriginAllowed(origin) { // check whether this origin is allowed + if trace { + traceLogger.Printf("HTTP Origin:%s is not part of %v, neither matches any part of %v", origin, c.AllowedDomains, c.allowedOriginPatterns) + } + chain.ProcessFilter(req, resp) + return + } + if req.Request.Method != "OPTIONS" { + c.doActualRequest(req, resp) + chain.ProcessFilter(req, resp) + return + } + if acrm := req.Request.Header.Get(HEADER_AccessControlRequestMethod); acrm != "" { + c.doPreflightRequest(req, resp) + } else { + c.doActualRequest(req, resp) + chain.ProcessFilter(req, resp) + return + } +} + +func (c CrossOriginResourceSharing) doActualRequest(req *Request, resp *Response) { + c.setOptionsHeaders(req, resp) + // continue processing the response +} + +func (c *CrossOriginResourceSharing) doPreflightRequest(req *Request, resp *Response) { + if len(c.AllowedMethods) == 0 { + if c.Container == nil { + c.AllowedMethods = DefaultContainer.computeAllowedMethods(req) + } else { + c.AllowedMethods = c.Container.computeAllowedMethods(req) + } + } + + acrm := req.Request.Header.Get(HEADER_AccessControlRequestMethod) + if !c.isValidAccessControlRequestMethod(acrm, c.AllowedMethods) { + if trace { + traceLogger.Printf("Http header %s:%s is not in %v", + HEADER_AccessControlRequestMethod, + acrm, + c.AllowedMethods) + } + return + } + acrhs := req.Request.Header.Get(HEADER_AccessControlRequestHeaders) + if len(acrhs) > 0 { + for _, each := range strings.Split(acrhs, ",") { + if !c.isValidAccessControlRequestHeader(strings.Trim(each, " ")) { + if trace { + traceLogger.Printf("Http header %s:%s is not in %v", + HEADER_AccessControlRequestHeaders, + acrhs, + c.AllowedHeaders) + } + return + } + } + } + resp.AddHeader(HEADER_AccessControlAllowMethods, strings.Join(c.AllowedMethods, ",")) + resp.AddHeader(HEADER_AccessControlAllowHeaders, acrhs) + c.setOptionsHeaders(req, resp) + + // return http 200 response, no body +} + +func (c CrossOriginResourceSharing) setOptionsHeaders(req *Request, resp *Response) { + c.checkAndSetExposeHeaders(resp) + c.setAllowOriginHeader(req, resp) + c.checkAndSetAllowCredentials(resp) + if c.MaxAge > 0 { + resp.AddHeader(HEADER_AccessControlMaxAge, strconv.Itoa(c.MaxAge)) + } +} + +func (c CrossOriginResourceSharing) isOriginAllowed(origin string) bool { + if len(origin) == 0 { + return false + } + if len(c.AllowedDomains) == 0 { + return true + } + + allowed := false + for _, domain := range c.AllowedDomains { + if domain == origin { + allowed = true + break + } + } + + if !allowed { + if len(c.allowedOriginPatterns) == 0 { + // compile allowed domains to allowed origin patterns + allowedOriginRegexps, err := compileRegexps(c.AllowedDomains) + if err != nil { + return false + } + c.allowedOriginPatterns = allowedOriginRegexps + } + + for _, pattern := range c.allowedOriginPatterns { + if allowed = pattern.MatchString(origin); allowed { + break + } + } + } + + return allowed +} + +func (c CrossOriginResourceSharing) setAllowOriginHeader(req *Request, resp *Response) { + origin := req.Request.Header.Get(HEADER_Origin) + if c.isOriginAllowed(origin) { + resp.AddHeader(HEADER_AccessControlAllowOrigin, origin) + } +} + +func (c CrossOriginResourceSharing) checkAndSetExposeHeaders(resp *Response) { + if len(c.ExposeHeaders) > 0 { + resp.AddHeader(HEADER_AccessControlExposeHeaders, strings.Join(c.ExposeHeaders, ",")) + } +} + +func (c CrossOriginResourceSharing) checkAndSetAllowCredentials(resp *Response) { + if c.CookiesAllowed { + resp.AddHeader(HEADER_AccessControlAllowCredentials, "true") + } +} + +func (c CrossOriginResourceSharing) isValidAccessControlRequestMethod(method string, allowedMethods []string) bool { + for _, each := range allowedMethods { + if each == method { + return true + } + } + return false +} + +func (c CrossOriginResourceSharing) isValidAccessControlRequestHeader(header string) bool { + for _, each := range c.AllowedHeaders { + if strings.ToLower(each) == strings.ToLower(header) { + return true + } + } + return false +} + +// Take a list of strings and compile them into a list of regular expressions. +func compileRegexps(regexpStrings []string) ([]*regexp.Regexp, error) { + regexps := []*regexp.Regexp{} + for _, regexpStr := range regexpStrings { + r, err := regexp.Compile(regexpStr) + if err != nil { + return regexps, err + } + regexps = append(regexps, r) + } + return regexps, nil +} diff --git a/vendor/github.com/emicklei/go-restful/cors_filter_test.go b/vendor/github.com/emicklei/go-restful/cors_filter_test.go new file mode 100644 index 00000000000..09c5d3300e8 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/cors_filter_test.go @@ -0,0 +1,129 @@ +package restful + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +// go test -v -test.run TestCORSFilter_Preflight ...restful +// http://www.html5rocks.com/en/tutorials/cors/#toc-handling-a-not-so-simple-request +func TestCORSFilter_Preflight(t *testing.T) { + tearDown() + ws := new(WebService) + ws.Route(ws.PUT("/cors").To(dummy)) + Add(ws) + + cors := CrossOriginResourceSharing{ + ExposeHeaders: []string{"X-Custom-Header"}, + AllowedHeaders: []string{"X-Custom-Header", "X-Additional-Header"}, + CookiesAllowed: true, + Container: DefaultContainer} + Filter(cors.Filter) + + // Preflight + httpRequest, _ := http.NewRequest("OPTIONS", "http://api.alice.com/cors", nil) + httpRequest.Method = "OPTIONS" + httpRequest.Header.Set(HEADER_Origin, "http://api.bob.com") + httpRequest.Header.Set(HEADER_AccessControlRequestMethod, "PUT") + httpRequest.Header.Set(HEADER_AccessControlRequestHeaders, "X-Custom-Header, X-Additional-Header") + + httpWriter := httptest.NewRecorder() + DefaultContainer.Dispatch(httpWriter, httpRequest) + + actual := httpWriter.Header().Get(HEADER_AccessControlAllowOrigin) + if "http://api.bob.com" != actual { + t.Fatal("expected: http://api.bob.com but got:" + actual) + } + actual = httpWriter.Header().Get(HEADER_AccessControlAllowMethods) + if "PUT" != actual { + t.Fatal("expected: PUT but got:" + actual) + } + actual = httpWriter.Header().Get(HEADER_AccessControlAllowHeaders) + if "X-Custom-Header, X-Additional-Header" != actual { + t.Fatal("expected: X-Custom-Header, X-Additional-Header but got:" + actual) + } + + if !cors.isOriginAllowed("somewhere") { + t.Fatal("origin expected to be allowed") + } + cors.AllowedDomains = []string{"overthere.com"} + if cors.isOriginAllowed("somewhere") { + t.Fatal("origin [somewhere] expected NOT to be allowed") + } + if !cors.isOriginAllowed("overthere.com") { + t.Fatal("origin [overthere] expected to be allowed") + } + +} + +// go test -v -test.run TestCORSFilter_Actual ...restful +// http://www.html5rocks.com/en/tutorials/cors/#toc-handling-a-not-so-simple-request +func TestCORSFilter_Actual(t *testing.T) { + tearDown() + ws := new(WebService) + ws.Route(ws.PUT("/cors").To(dummy)) + Add(ws) + + cors := CrossOriginResourceSharing{ + ExposeHeaders: []string{"X-Custom-Header"}, + AllowedHeaders: []string{"X-Custom-Header", "X-Additional-Header"}, + CookiesAllowed: true, + Container: DefaultContainer} + Filter(cors.Filter) + + // Actual + httpRequest, _ := http.NewRequest("PUT", "http://api.alice.com/cors", nil) + httpRequest.Header.Set(HEADER_Origin, "http://api.bob.com") + httpRequest.Header.Set("X-Custom-Header", "value") + + httpWriter := httptest.NewRecorder() + DefaultContainer.Dispatch(httpWriter, httpRequest) + actual := httpWriter.Header().Get(HEADER_AccessControlAllowOrigin) + if "http://api.bob.com" != actual { + t.Fatal("expected: http://api.bob.com but got:" + actual) + } + if httpWriter.Body.String() != "dummy" { + t.Fatal("expected: dummy but got:" + httpWriter.Body.String()) + } +} + +var allowedDomainInput = []struct { + domains []string + origin string + allowed bool +}{ + {[]string{}, "http://anything.com", true}, + {[]string{"example.com"}, "example.com", true}, + {[]string{"example.com"}, "not-allowed", false}, + {[]string{"not-matching.com", "example.com"}, "example.com", true}, + {[]string{".*"}, "example.com", true}, +} + +// go test -v -test.run TestCORSFilter_AllowedDomains ...restful +func TestCORSFilter_AllowedDomains(t *testing.T) { + for _, each := range allowedDomainInput { + tearDown() + ws := new(WebService) + ws.Route(ws.PUT("/cors").To(dummy)) + Add(ws) + + cors := CrossOriginResourceSharing{ + AllowedDomains: each.domains, + CookiesAllowed: true, + Container: DefaultContainer} + Filter(cors.Filter) + + httpRequest, _ := http.NewRequest("PUT", "http://api.his.com/cors", nil) + httpRequest.Header.Set(HEADER_Origin, each.origin) + httpWriter := httptest.NewRecorder() + DefaultContainer.Dispatch(httpWriter, httpRequest) + actual := httpWriter.Header().Get(HEADER_AccessControlAllowOrigin) + if actual != each.origin && each.allowed { + t.Fatal("expected to be accepted") + } + if actual == each.origin && !each.allowed { + t.Fatal("did not expect to be accepted") + } + } +} diff --git a/vendor/github.com/emicklei/go-restful/coverage.sh b/vendor/github.com/emicklei/go-restful/coverage.sh new file mode 100644 index 00000000000..e27dbf1a913 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/coverage.sh @@ -0,0 +1,2 @@ +go test -coverprofile=coverage.out +go tool cover -html=coverage.out \ No newline at end of file diff --git a/vendor/github.com/emicklei/go-restful/curly.go b/vendor/github.com/emicklei/go-restful/curly.go new file mode 100644 index 00000000000..79f1f5aa200 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/curly.go @@ -0,0 +1,164 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "net/http" + "regexp" + "sort" + "strings" +) + +// CurlyRouter expects Routes with paths that contain zero or more parameters in curly brackets. +type CurlyRouter struct{} + +// SelectRoute is part of the Router interface and returns the best match +// for the WebService and its Route for the given Request. +func (c CurlyRouter) SelectRoute( + webServices []*WebService, + httpRequest *http.Request) (selectedService *WebService, selected *Route, err error) { + + requestTokens := tokenizePath(httpRequest.URL.Path) + + detectedService := c.detectWebService(requestTokens, webServices) + if detectedService == nil { + if trace { + traceLogger.Printf("no WebService was found to match URL path:%s\n", httpRequest.URL.Path) + } + return nil, nil, NewError(http.StatusNotFound, "404: Page Not Found") + } + candidateRoutes := c.selectRoutes(detectedService, requestTokens) + if len(candidateRoutes) == 0 { + if trace { + traceLogger.Printf("no Route in WebService with path %s was found to match URL path:%s\n", detectedService.rootPath, httpRequest.URL.Path) + } + return detectedService, nil, NewError(http.StatusNotFound, "404: Page Not Found") + } + selectedRoute, err := c.detectRoute(candidateRoutes, httpRequest) + if selectedRoute == nil { + return detectedService, nil, err + } + return detectedService, selectedRoute, nil +} + +// selectRoutes return a collection of Route from a WebService that matches the path tokens from the request. +func (c CurlyRouter) selectRoutes(ws *WebService, requestTokens []string) sortableCurlyRoutes { + candidates := sortableCurlyRoutes{} + for _, each := range ws.routes { + matches, paramCount, staticCount := c.matchesRouteByPathTokens(each.pathParts, requestTokens) + if matches { + candidates.add(curlyRoute{each, paramCount, staticCount}) // TODO make sure Routes() return pointers? + } + } + sort.Sort(sort.Reverse(candidates)) + return candidates +} + +// matchesRouteByPathTokens computes whether it matches, howmany parameters do match and what the number of static path elements are. +func (c CurlyRouter) matchesRouteByPathTokens(routeTokens, requestTokens []string) (matches bool, paramCount int, staticCount int) { + if len(routeTokens) < len(requestTokens) { + // proceed in matching only if last routeToken is wildcard + count := len(routeTokens) + if count == 0 || !strings.HasSuffix(routeTokens[count-1], "*}") { + return false, 0, 0 + } + // proceed + } + for i, routeToken := range routeTokens { + if i == len(requestTokens) { + // reached end of request path + return false, 0, 0 + } + requestToken := requestTokens[i] + if strings.HasPrefix(routeToken, "{") { + paramCount++ + if colon := strings.Index(routeToken, ":"); colon != -1 { + // match by regex + matchesToken, matchesRemainder := c.regularMatchesPathToken(routeToken, colon, requestToken) + if !matchesToken { + return false, 0, 0 + } + if matchesRemainder { + break + } + } + } else { // no { prefix + if requestToken != routeToken { + return false, 0, 0 + } + staticCount++ + } + } + return true, paramCount, staticCount +} + +// regularMatchesPathToken tests whether the regular expression part of routeToken matches the requestToken or all remaining tokens +// format routeToken is {someVar:someExpression}, e.g. {zipcode:[\d][\d][\d][\d][A-Z][A-Z]} +func (c CurlyRouter) regularMatchesPathToken(routeToken string, colon int, requestToken string) (matchesToken bool, matchesRemainder bool) { + regPart := routeToken[colon+1 : len(routeToken)-1] + if regPart == "*" { + if trace { + traceLogger.Printf("wildcard parameter detected in route token %s that matches %s\n", routeToken, requestToken) + } + return true, true + } + matched, err := regexp.MatchString(regPart, requestToken) + return (matched && err == nil), false +} + +var jsr311Router = RouterJSR311{} + +// detectRoute selectes from a list of Route the first match by inspecting both the Accept and Content-Type +// headers of the Request. See also RouterJSR311 in jsr311.go +func (c CurlyRouter) detectRoute(candidateRoutes sortableCurlyRoutes, httpRequest *http.Request) (*Route, error) { + // tracing is done inside detectRoute + return jsr311Router.detectRoute(candidateRoutes.routes(), httpRequest) +} + +// detectWebService returns the best matching webService given the list of path tokens. +// see also computeWebserviceScore +func (c CurlyRouter) detectWebService(requestTokens []string, webServices []*WebService) *WebService { + var best *WebService + score := -1 + for _, each := range webServices { + matches, eachScore := c.computeWebserviceScore(requestTokens, each.pathExpr.tokens) + if matches && (eachScore > score) { + best = each + score = eachScore + } + } + return best +} + +// computeWebserviceScore returns whether tokens match and +// the weighted score of the longest matching consecutive tokens from the beginning. +func (c CurlyRouter) computeWebserviceScore(requestTokens []string, tokens []string) (bool, int) { + if len(tokens) > len(requestTokens) { + return false, 0 + } + score := 0 + for i := 0; i < len(tokens); i++ { + each := requestTokens[i] + other := tokens[i] + if len(each) == 0 && len(other) == 0 { + score++ + continue + } + if len(other) > 0 && strings.HasPrefix(other, "{") { + // no empty match + if len(each) == 0 { + return false, score + } + score += 1 + } else { + // not a parameter + if each != other { + return false, score + } + score += (len(tokens) - i) * 10 //fuzzy + } + } + return true, score +} diff --git a/vendor/github.com/emicklei/go-restful/curly_route.go b/vendor/github.com/emicklei/go-restful/curly_route.go new file mode 100644 index 00000000000..296f94650e6 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/curly_route.go @@ -0,0 +1,52 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +// curlyRoute exits for sorting Routes by the CurlyRouter based on number of parameters and number of static path elements. +type curlyRoute struct { + route Route + paramCount int + staticCount int +} + +type sortableCurlyRoutes []curlyRoute + +func (s *sortableCurlyRoutes) add(route curlyRoute) { + *s = append(*s, route) +} + +func (s sortableCurlyRoutes) routes() (routes []Route) { + for _, each := range s { + routes = append(routes, each.route) // TODO change return type + } + return routes +} + +func (s sortableCurlyRoutes) Len() int { + return len(s) +} +func (s sortableCurlyRoutes) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} +func (s sortableCurlyRoutes) Less(i, j int) bool { + ci := s[i] + cj := s[j] + + // primary key + if ci.staticCount < cj.staticCount { + return true + } + if ci.staticCount > cj.staticCount { + return false + } + // secundary key + if ci.paramCount < cj.paramCount { + return true + } + if ci.paramCount > cj.paramCount { + return false + } + return ci.route.Path < cj.route.Path +} diff --git a/vendor/github.com/emicklei/go-restful/curly_test.go b/vendor/github.com/emicklei/go-restful/curly_test.go new file mode 100644 index 00000000000..0568dfac10e --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/curly_test.go @@ -0,0 +1,238 @@ +package restful + +import ( + "io" + "net/http" + "testing" +) + +var requestPaths = []struct { + // url with path (1) is handled by service with root (2) and remainder has value final (3) + path, root string +}{ + {"/", "/"}, + {"/p", "/p"}, + {"/p/x", "/p/{q}"}, + {"/q/x", "/q"}, + {"/p/x/", "/p/{q}"}, + {"/p/x/y", "/p/{q}"}, + {"/q/x/y", "/q"}, + {"/z/q", "/{p}/q"}, + {"/a/b/c/q", "/"}, +} + +// go test -v -test.run TestCurlyDetectWebService ...restful +func TestCurlyDetectWebService(t *testing.T) { + ws1 := new(WebService).Path("/") + ws2 := new(WebService).Path("/p") + ws3 := new(WebService).Path("/q") + ws4 := new(WebService).Path("/p/q") + ws5 := new(WebService).Path("/p/{q}") + ws7 := new(WebService).Path("/{p}/q") + var wss = []*WebService{ws1, ws2, ws3, ws4, ws5, ws7} + + for _, each := range wss { + t.Logf("path=%s,toks=%v\n", each.pathExpr.Source, each.pathExpr.tokens) + } + + router := CurlyRouter{} + + ok := true + for i, fixture := range requestPaths { + requestTokens := tokenizePath(fixture.path) + who := router.detectWebService(requestTokens, wss) + if who != nil && who.RootPath() != fixture.root { + t.Logf("[line:%v] Unexpected dispatcher, expected:%v, actual:%v", i, fixture.root, who.RootPath()) + ok = false + } + } + if !ok { + t.Fail() + } +} + +var serviceDetects = []struct { + path string + found bool + root string +}{ + {"/a/b", true, "/{p}/{q}/{r}"}, + {"/p/q", true, "/p/q"}, + {"/q/p", true, "/q"}, + {"/", true, "/"}, + {"/p/q/r", true, "/p/q"}, +} + +// go test -v -test.run Test_detectWebService ...restful +func Test_detectWebService(t *testing.T) { + router := CurlyRouter{} + ws1 := new(WebService).Path("/") + ws2 := new(WebService).Path("/p") + ws3 := new(WebService).Path("/q") + ws4 := new(WebService).Path("/p/q") + ws5 := new(WebService).Path("/p/{q}") + ws6 := new(WebService).Path("/p/{q}/") + ws7 := new(WebService).Path("/{p}/q") + ws8 := new(WebService).Path("/{p}/{q}/{r}") + var wss = []*WebService{ws8, ws7, ws6, ws5, ws4, ws3, ws2, ws1} + for _, fix := range serviceDetects { + requestPath := fix.path + requestTokens := tokenizePath(requestPath) + for _, ws := range wss { + serviceTokens := ws.pathExpr.tokens + matches, score := router.computeWebserviceScore(requestTokens, serviceTokens) + t.Logf("req=%s,toks:%v,ws=%s,toks:%v,score=%d,matches=%v", requestPath, requestTokens, ws.RootPath(), serviceTokens, score, matches) + } + best := router.detectWebService(requestTokens, wss) + if best != nil { + if fix.found { + t.Logf("best=%s", best.RootPath()) + } else { + t.Fatalf("should have found:%s", fix.root) + } + } + } +} + +var routeMatchers = []struct { + route string + path string + matches bool + paramCount int + staticCount int +}{ + // route, request-path + {"/a", "/a", true, 0, 1}, + {"/a", "/b", false, 0, 0}, + {"/a", "/b", false, 0, 0}, + {"/a/{b}/c/", "/a/2/c", true, 1, 2}, + {"/{a}/{b}/{c}/", "/a/b", false, 0, 0}, + {"/{x:*}", "/", false, 0, 0}, + {"/{x:*}", "/a", true, 1, 0}, + {"/{x:*}", "/a/b", true, 1, 0}, + {"/a/{x:*}", "/a/b", true, 1, 1}, + {"/a/{x:[A-Z][A-Z]}", "/a/ZX", true, 1, 1}, + {"/basepath/{resource:*}", "/basepath/some/other/location/test.xml", true, 1, 1}, +} + +// clear && go test -v -test.run Test_matchesRouteByPathTokens ...restful +func Test_matchesRouteByPathTokens(t *testing.T) { + router := CurlyRouter{} + for i, each := range routeMatchers { + routeToks := tokenizePath(each.route) + reqToks := tokenizePath(each.path) + matches, pCount, sCount := router.matchesRouteByPathTokens(routeToks, reqToks) + if matches != each.matches { + t.Fatalf("[%d] unexpected matches outcome route:%s, path:%s, matches:%v", i, each.route, each.path, matches) + } + if pCount != each.paramCount { + t.Fatalf("[%d] unexpected paramCount got:%d want:%d ", i, pCount, each.paramCount) + } + if sCount != each.staticCount { + t.Fatalf("[%d] unexpected staticCount got:%d want:%d ", i, sCount, each.staticCount) + } + } +} + +// clear && go test -v -test.run TestExtractParameters_Wildcard1 ...restful +func TestExtractParameters_Wildcard1(t *testing.T) { + params := doExtractParams("/fixed/{var:*}", 2, "/fixed/remainder", t) + if params["var"] != "remainder" { + t.Errorf("parameter mismatch var: %s", params["var"]) + } +} + +// clear && go test -v -test.run TestExtractParameters_Wildcard2 ...restful +func TestExtractParameters_Wildcard2(t *testing.T) { + params := doExtractParams("/fixed/{var:*}", 2, "/fixed/remain/der", t) + if params["var"] != "remain/der" { + t.Errorf("parameter mismatch var: %s", params["var"]) + } +} + +// clear && go test -v -test.run TestExtractParameters_Wildcard3 ...restful +func TestExtractParameters_Wildcard3(t *testing.T) { + params := doExtractParams("/static/{var:*}", 2, "/static/test/sub/hi.html", t) + if params["var"] != "test/sub/hi.html" { + t.Errorf("parameter mismatch var: %s", params["var"]) + } +} + +func TestExtractParameters_Wildcard4(t *testing.T) { + params := doExtractParams("/static/{var:*}/sub", 3, "/static/test/sub", t) + if params["var"] != "test/sub" { + t.Errorf("parameter mismatch var: %s", params["var"]) + } +} + +// clear && go test -v -test.run TestCurly_ISSUE_34 ...restful +func TestCurly_ISSUE_34(t *testing.T) { + ws1 := new(WebService).Path("/") + ws1.Route(ws1.GET("/{type}/{id}").To(curlyDummy)) + ws1.Route(ws1.GET("/network/{id}").To(curlyDummy)) + croutes := CurlyRouter{}.selectRoutes(ws1, tokenizePath("/network/12")) + if len(croutes) != 2 { + t.Fatal("expected 2 routes") + } + if got, want := croutes[0].route.Path, "/network/{id}"; got != want { + t.Errorf("got %v want %v", got, want) + } +} + +// clear && go test -v -test.run TestCurly_ISSUE_34_2 ...restful +func TestCurly_ISSUE_34_2(t *testing.T) { + ws1 := new(WebService) + ws1.Route(ws1.GET("/network/{id}").To(curlyDummy)) + ws1.Route(ws1.GET("/{type}/{id}").To(curlyDummy)) + croutes := CurlyRouter{}.selectRoutes(ws1, tokenizePath("/network/12")) + if len(croutes) != 2 { + t.Fatal("expected 2 routes") + } + if got, want := croutes[0].route.Path, "/network/{id}"; got != want { + t.Errorf("got %v want %v", got, want) + } +} + +// clear && go test -v -test.run TestCurly_JsonHtml ...restful +func TestCurly_JsonHtml(t *testing.T) { + ws1 := new(WebService) + ws1.Path("/") + ws1.Route(ws1.GET("/some.html").To(curlyDummy).Consumes("*/*").Produces("text/html")) + req, _ := http.NewRequest("GET", "/some.html", nil) + req.Header.Set("Accept", "application/json") + _, route, err := CurlyRouter{}.SelectRoute([]*WebService{ws1}, req) + if err == nil { + t.Error("error expected") + } + if route != nil { + t.Error("no route expected") + } +} + +// go test -v -test.run TestCurly_ISSUE_137 ...restful +func TestCurly_ISSUE_137(t *testing.T) { + ws1 := new(WebService) + ws1.Route(ws1.GET("/hello").To(curlyDummy)) + ws1.Path("/") + req, _ := http.NewRequest("GET", "/", nil) + _, route, _ := CurlyRouter{}.SelectRoute([]*WebService{ws1}, req) + t.Log(route) + if route != nil { + t.Error("no route expected") + } +} + +// go test -v -test.run TestCurly_ISSUE_137_2 ...restful +func TestCurly_ISSUE_137_2(t *testing.T) { + ws1 := new(WebService) + ws1.Route(ws1.GET("/hello").To(curlyDummy)) + ws1.Path("/") + req, _ := http.NewRequest("GET", "/hello/bob", nil) + _, route, _ := CurlyRouter{}.SelectRoute([]*WebService{ws1}, req) + t.Log(route) + if route != nil { + t.Errorf("no route expected, got %v", route) + } +} + +func curlyDummy(req *Request, resp *Response) { io.WriteString(resp.ResponseWriter, "curlyDummy") } diff --git a/vendor/github.com/emicklei/go-restful/doc.go b/vendor/github.com/emicklei/go-restful/doc.go new file mode 100644 index 00000000000..f7c16b01fee --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/doc.go @@ -0,0 +1,185 @@ +/* +Package restful , a lean package for creating REST-style WebServices without magic. + +WebServices and Routes + +A WebService has a collection of Route objects that dispatch incoming Http Requests to a function calls. +Typically, a WebService has a root path (e.g. /users) and defines common MIME types for its routes. +WebServices must be added to a container (see below) in order to handler Http requests from a server. + +A Route is defined by a HTTP method, an URL path and (optionally) the MIME types it consumes (Content-Type) and produces (Accept). +This package has the logic to find the best matching Route and if found, call its Function. + + ws := new(restful.WebService) + ws. + Path("/users"). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON, restful.MIME_XML) + + ws.Route(ws.GET("/{user-id}").To(u.findUser)) // u is a UserResource + + ... + + // GET http://localhost:8080/users/1 + func (u UserResource) findUser(request *restful.Request, response *restful.Response) { + id := request.PathParameter("user-id") + ... + } + +The (*Request, *Response) arguments provide functions for reading information from the request and writing information back to the response. + +See the example https://github.com/emicklei/go-restful/blob/master/examples/restful-user-resource.go with a full implementation. + +Regular expression matching Routes + +A Route parameter can be specified using the format "uri/{var[:regexp]}" or the special version "uri/{var:*}" for matching the tail of the path. +For example, /persons/{name:[A-Z][A-Z]} can be used to restrict values for the parameter "name" to only contain capital alphabetic characters. +Regular expressions must use the standard Go syntax as described in the regexp package. (https://code.google.com/p/re2/wiki/Syntax) +This feature requires the use of a CurlyRouter. + +Containers + +A Container holds a collection of WebServices, Filters and a http.ServeMux for multiplexing http requests. +Using the statements "restful.Add(...) and restful.Filter(...)" will register WebServices and Filters to the Default Container. +The Default container of go-restful uses the http.DefaultServeMux. +You can create your own Container and create a new http.Server for that particular container. + + container := restful.NewContainer() + server := &http.Server{Addr: ":8081", Handler: container} + +Filters + +A filter dynamically intercepts requests and responses to transform or use the information contained in the requests or responses. +You can use filters to perform generic logging, measurement, authentication, redirect, set response headers etc. +In the restful package there are three hooks into the request,response flow where filters can be added. +Each filter must define a FilterFunction: + + func (req *restful.Request, resp *restful.Response, chain *restful.FilterChain) + +Use the following statement to pass the request,response pair to the next filter or RouteFunction + + chain.ProcessFilter(req, resp) + +Container Filters + +These are processed before any registered WebService. + + // install a (global) filter for the default container (processed before any webservice) + restful.Filter(globalLogging) + +WebService Filters + +These are processed before any Route of a WebService. + + // install a webservice filter (processed before any route) + ws.Filter(webserviceLogging).Filter(measureTime) + + +Route Filters + +These are processed before calling the function associated with the Route. + + // install 2 chained route filters (processed before calling findUser) + ws.Route(ws.GET("/{user-id}").Filter(routeLogging).Filter(NewCountFilter().routeCounter).To(findUser)) + +See the example https://github.com/emicklei/go-restful/blob/master/examples/restful-filters.go with full implementations. + +Response Encoding + +Two encodings are supported: gzip and deflate. To enable this for all responses: + + restful.DefaultContainer.EnableContentEncoding(true) + +If a Http request includes the Accept-Encoding header then the response content will be compressed using the specified encoding. +Alternatively, you can create a Filter that performs the encoding and install it per WebService or Route. + +See the example https://github.com/emicklei/go-restful/blob/master/examples/restful-encoding-filter.go + +OPTIONS support + +By installing a pre-defined container filter, your Webservice(s) can respond to the OPTIONS Http request. + + Filter(OPTIONSFilter()) + +CORS + +By installing the filter of a CrossOriginResourceSharing (CORS), your WebService(s) can handle CORS requests. + + cors := CrossOriginResourceSharing{ExposeHeaders: []string{"X-My-Header"}, CookiesAllowed: false, Container: DefaultContainer} + Filter(cors.Filter) + +Error Handling + +Unexpected things happen. If a request cannot be processed because of a failure, your service needs to tell via the response what happened and why. +For this reason HTTP status codes exist and it is important to use the correct code in every exceptional situation. + + 400: Bad Request + +If path or query parameters are not valid (content or type) then use http.StatusBadRequest. + + 404: Not Found + +Despite a valid URI, the resource requested may not be available + + 500: Internal Server Error + +If the application logic could not process the request (or write the response) then use http.StatusInternalServerError. + + 405: Method Not Allowed + +The request has a valid URL but the method (GET,PUT,POST,...) is not allowed. + + 406: Not Acceptable + +The request does not have or has an unknown Accept Header set for this operation. + + 415: Unsupported Media Type + +The request does not have or has an unknown Content-Type Header set for this operation. + +ServiceError + +In addition to setting the correct (error) Http status code, you can choose to write a ServiceError message on the response. + +Performance options + +This package has several options that affect the performance of your service. It is important to understand them and how you can change it. + + restful.DefaultContainer.DoNotRecover(false) + +DoNotRecover controls whether panics will be caught to return HTTP 500. +If set to false, the container will recover from panics. +Default value is true + + restful.SetCompressorProvider(NewBoundedCachedCompressors(20, 20)) + +If content encoding is enabled then the default strategy for getting new gzip/zlib writers and readers is to use a sync.Pool. +Because writers are expensive structures, performance is even more improved when using a preloaded cache. You can also inject your own implementation. + +Trouble shooting + +This package has the means to produce detail logging of the complete Http request matching process and filter invocation. +Enabling this feature requires you to set an implementation of restful.StdLogger (e.g. log.Logger) instance such as: + + restful.TraceLogger(log.New(os.Stdout, "[restful] ", log.LstdFlags|log.Lshortfile)) + +Logging + +The restful.SetLogger() method allows you to override the logger used by the package. By default restful +uses the standard library `log` package and logs to stdout. Different logging packages are supported as +long as they conform to `StdLogger` interface defined in the `log` sub-package, writing an adapter for your +preferred package is simple. + +Resources + +[project]: https://github.com/emicklei/go-restful + +[examples]: https://github.com/emicklei/go-restful/blob/master/examples + +[design]: http://ernestmicklei.com/2012/11/11/go-restful-api-design/ + +[showcases]: https://github.com/emicklei/mora, https://github.com/emicklei/landskape + +(c) 2012-2015, http://ernestmicklei.com. MIT License +*/ +package restful diff --git a/vendor/github.com/emicklei/go-restful/doc_examples_test.go b/vendor/github.com/emicklei/go-restful/doc_examples_test.go new file mode 100644 index 00000000000..0af636e5536 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/doc_examples_test.go @@ -0,0 +1,41 @@ +package restful + +import "net/http" + +func ExampleOPTIONSFilter() { + // Install the OPTIONS filter on the default Container + Filter(OPTIONSFilter()) +} +func ExampleContainer_OPTIONSFilter() { + // Install the OPTIONS filter on a Container + myContainer := new(Container) + myContainer.Filter(myContainer.OPTIONSFilter) +} + +func ExampleContainer() { + // The Default container of go-restful uses the http.DefaultServeMux. + // You can create your own Container using restful.NewContainer() and create a new http.Server for that particular container + + ws := new(WebService) + wsContainer := NewContainer() + wsContainer.Add(ws) + server := &http.Server{Addr: ":8080", Handler: wsContainer} + server.ListenAndServe() +} + +func ExampleCrossOriginResourceSharing() { + // To install this filter on the Default Container use: + cors := CrossOriginResourceSharing{ExposeHeaders: []string{"X-My-Header"}, CookiesAllowed: false, Container: DefaultContainer} + Filter(cors.Filter) +} + +func ExampleServiceError() { + resp := new(Response) + resp.WriteEntity(NewError(http.StatusBadRequest, "Non-integer {id} path parameter")) +} + +func ExampleBoundedCachedCompressors() { + // Register a compressor provider (gzip/deflate read/write) that uses + // a bounded cache with a maximum of 20 writers and 20 readers. + SetCompressorProvider(NewBoundedCachedCompressors(20, 20)) +} diff --git a/vendor/github.com/emicklei/go-restful/entity_accessors.go b/vendor/github.com/emicklei/go-restful/entity_accessors.go new file mode 100644 index 00000000000..66dfc824f55 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/entity_accessors.go @@ -0,0 +1,162 @@ +package restful + +// Copyright 2015 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "encoding/xml" + "strings" + "sync" +) + +// EntityReaderWriter can read and write values using an encoding such as JSON,XML. +type EntityReaderWriter interface { + // Read a serialized version of the value from the request. + // The Request may have a decompressing reader. Depends on Content-Encoding. + Read(req *Request, v interface{}) error + + // Write a serialized version of the value on the response. + // The Response may have a compressing writer. Depends on Accept-Encoding. + // status should be a valid Http Status code + Write(resp *Response, status int, v interface{}) error +} + +// entityAccessRegistry is a singleton +var entityAccessRegistry = &entityReaderWriters{ + protection: new(sync.RWMutex), + accessors: map[string]EntityReaderWriter{}, +} + +// entityReaderWriters associates MIME to an EntityReaderWriter +type entityReaderWriters struct { + protection *sync.RWMutex + accessors map[string]EntityReaderWriter +} + +func init() { + RegisterEntityAccessor(MIME_JSON, NewEntityAccessorJSON(MIME_JSON)) + RegisterEntityAccessor(MIME_XML, NewEntityAccessorXML(MIME_XML)) +} + +// RegisterEntityAccessor add/overrides the ReaderWriter for encoding content with this MIME type. +func RegisterEntityAccessor(mime string, erw EntityReaderWriter) { + entityAccessRegistry.protection.Lock() + defer entityAccessRegistry.protection.Unlock() + entityAccessRegistry.accessors[mime] = erw +} + +// NewEntityAccessorJSON returns a new EntityReaderWriter for accessing JSON content. +// This package is already initialized with such an accessor using the MIME_JSON contentType. +func NewEntityAccessorJSON(contentType string) EntityReaderWriter { + return entityJSONAccess{ContentType: contentType} +} + +// NewEntityAccessorXML returns a new EntityReaderWriter for accessing XML content. +// This package is already initialized with such an accessor using the MIME_XML contentType. +func NewEntityAccessorXML(contentType string) EntityReaderWriter { + return entityXMLAccess{ContentType: contentType} +} + +// accessorAt returns the registered ReaderWriter for this MIME type. +func (r *entityReaderWriters) accessorAt(mime string) (EntityReaderWriter, bool) { + r.protection.RLock() + defer r.protection.RUnlock() + er, ok := r.accessors[mime] + if !ok { + // retry with reverse lookup + // more expensive but we are in an exceptional situation anyway + for k, v := range r.accessors { + if strings.Contains(mime, k) { + return v, true + } + } + } + return er, ok +} + +// entityXMLAccess is a EntityReaderWriter for XML encoding +type entityXMLAccess struct { + // This is used for setting the Content-Type header when writing + ContentType string +} + +// Read unmarshalls the value from XML +func (e entityXMLAccess) Read(req *Request, v interface{}) error { + return xml.NewDecoder(req.Request.Body).Decode(v) +} + +// Write marshalls the value to JSON and set the Content-Type Header. +func (e entityXMLAccess) Write(resp *Response, status int, v interface{}) error { + return writeXML(resp, status, e.ContentType, v) +} + +// writeXML marshalls the value to JSON and set the Content-Type Header. +func writeXML(resp *Response, status int, contentType string, v interface{}) error { + if v == nil { + resp.WriteHeader(status) + // do not write a nil representation + return nil + } + if resp.prettyPrint { + // pretty output must be created and written explicitly + output, err := xml.MarshalIndent(v, " ", " ") + if err != nil { + return err + } + resp.Header().Set(HEADER_ContentType, contentType) + resp.WriteHeader(status) + _, err = resp.Write([]byte(xml.Header)) + if err != nil { + return err + } + _, err = resp.Write(output) + return err + } + // not-so-pretty + resp.Header().Set(HEADER_ContentType, contentType) + resp.WriteHeader(status) + return xml.NewEncoder(resp).Encode(v) +} + +// entityJSONAccess is a EntityReaderWriter for JSON encoding +type entityJSONAccess struct { + // This is used for setting the Content-Type header when writing + ContentType string +} + +// Read unmarshalls the value from JSON +func (e entityJSONAccess) Read(req *Request, v interface{}) error { + decoder := NewDecoder(req.Request.Body) + decoder.UseNumber() + return decoder.Decode(v) +} + +// Write marshalls the value to JSON and set the Content-Type Header. +func (e entityJSONAccess) Write(resp *Response, status int, v interface{}) error { + return writeJSON(resp, status, e.ContentType, v) +} + +// write marshalls the value to JSON and set the Content-Type Header. +func writeJSON(resp *Response, status int, contentType string, v interface{}) error { + if v == nil { + resp.WriteHeader(status) + // do not write a nil representation + return nil + } + if resp.prettyPrint { + // pretty output must be created and written explicitly + output, err := MarshalIndent(v, "", " ") + if err != nil { + return err + } + resp.Header().Set(HEADER_ContentType, contentType) + resp.WriteHeader(status) + _, err = resp.Write(output) + return err + } + // not-so-pretty + resp.Header().Set(HEADER_ContentType, contentType) + resp.WriteHeader(status) + return NewEncoder(resp).Encode(v) +} diff --git a/vendor/github.com/emicklei/go-restful/entity_accessors_test.go b/vendor/github.com/emicklei/go-restful/entity_accessors_test.go new file mode 100644 index 00000000000..d1c1e1585f7 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/entity_accessors_test.go @@ -0,0 +1,69 @@ +package restful + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "testing" +) + +type keyvalue struct { + readCalled bool + writeCalled bool +} + +func (kv *keyvalue) Read(req *Request, v interface{}) error { + //t := reflect.TypeOf(v) + //rv := reflect.ValueOf(v) + kv.readCalled = true + return nil +} + +func (kv *keyvalue) Write(resp *Response, status int, v interface{}) error { + t := reflect.TypeOf(v) + rv := reflect.ValueOf(v) + for ix := 0; ix < t.NumField(); ix++ { + sf := t.Field(ix) + io.WriteString(resp, sf.Name) + io.WriteString(resp, "=") + io.WriteString(resp, fmt.Sprintf("%v\n", rv.Field(ix).Interface())) + } + kv.writeCalled = true + return nil +} + +// go test -v -test.run TestKeyValueEncoding ...restful +func TestKeyValueEncoding(t *testing.T) { + type Book struct { + Title string + Author string + PublishedYear int + } + kv := new(keyvalue) + RegisterEntityAccessor("application/kv", kv) + b := Book{"Singing for Dummies", "john doe", 2015} + + // Write + httpWriter := httptest.NewRecorder() + // Accept Produces + resp := Response{ResponseWriter: httpWriter, requestAccept: "application/kv,*/*;q=0.8", routeProduces: []string{"application/kv"}, prettyPrint: true} + resp.WriteEntity(b) + t.Log(string(httpWriter.Body.Bytes())) + if !kv.writeCalled { + t.Error("Write never called") + } + + // Read + bodyReader := bytes.NewReader(httpWriter.Body.Bytes()) + httpRequest, _ := http.NewRequest("GET", "/test", bodyReader) + httpRequest.Header.Set("Content-Type", "application/kv; charset=UTF-8") + request := NewRequest(httpRequest) + var bb Book + request.ReadEntity(&bb) + if !kv.readCalled { + t.Error("Read never called") + } +} diff --git a/vendor/github.com/emicklei/go-restful/examples/.goconvey b/vendor/github.com/emicklei/go-restful/examples/.goconvey new file mode 100644 index 00000000000..8485e986e45 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/.goconvey @@ -0,0 +1 @@ +ignore \ No newline at end of file diff --git a/vendor/github.com/emicklei/go-restful/examples/google_app_engine/.goconvey b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/.goconvey new file mode 100644 index 00000000000..8485e986e45 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/.goconvey @@ -0,0 +1 @@ +ignore \ No newline at end of file diff --git a/vendor/github.com/emicklei/go-restful/examples/google_app_engine/app.yaml b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/app.yaml new file mode 100644 index 00000000000..362db6b078a --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/app.yaml @@ -0,0 +1,20 @@ +# +# Include your application ID here +# +application: +version: 1 +runtime: go +api_version: go1 + +handlers: +# +# Regex for all swagger files to make as static content. +# You should create the folder static/swagger and copy +# swagger-ui into it. +# +- url: /apidocs/(.*?)/(.*\.(js|html|css)) + static_files: static/swagger/\1/\2 + upload: static/swagger/(.*?)/(.*\.(js|html|css)) + +- url: /.* + script: _go_app diff --git a/vendor/github.com/emicklei/go-restful/examples/google_app_engine/datastore/.goconvey b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/datastore/.goconvey new file mode 100644 index 00000000000..8485e986e45 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/datastore/.goconvey @@ -0,0 +1 @@ +ignore \ No newline at end of file diff --git a/vendor/github.com/emicklei/go-restful/examples/google_app_engine/datastore/app.yaml b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/datastore/app.yaml new file mode 100644 index 00000000000..1ac9dca28f3 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/datastore/app.yaml @@ -0,0 +1,18 @@ +application: +version: 1 +runtime: go +api_version: go1 + +handlers: +# Regex for all swagger files to make as static content. +# You should create the folder static/swagger and copy +# swagger-ui into it. +# +- url: /apidocs/(.*?)/(.*\.(js|html|css)) + static_files: static/swagger/\1/\2 + upload: static/swagger/(.*?)/(.*\.(js|html|css)) + +# Catch all. +- url: /.* + script: _go_app + login: required diff --git a/vendor/github.com/emicklei/go-restful/examples/google_app_engine/datastore/main.go b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/datastore/main.go new file mode 100644 index 00000000000..33e5b2ea68a --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/datastore/main.go @@ -0,0 +1,267 @@ +package main + +import ( + "net/http" + "time" + + "github.com/emicklei/go-restful" + "github.com/emicklei/go-restful-swagger12" + "google.golang.org/appengine" + "google.golang.org/appengine/datastore" + "google.golang.org/appengine/user" +) + +// This example demonstrates a reasonably complete suite of RESTful operations backed +// by DataStore on Google App Engine. + +// Our simple example struct. +type Profile struct { + LastModified time.Time `json:"-" xml:"-"` + Email string `json:"-" xml:"-"` + FirstName string `json:"first_name" xml:"first-name"` + NickName string `json:"nick_name" xml:"nick-name"` + LastName string `json:"last_name" xml:"last-name"` +} + +type ProfileApi struct { + Path string +} + +func gaeUrl() string { + if appengine.IsDevAppServer() { + return "http://localhost:8080" + } else { + // Include your URL on App Engine here. + // I found no way to get AppID without appengine.Context and this always + // based on a http.Request. + return "http://federatedservices.appspot.com" + } +} + +func init() { + u := ProfileApi{Path: "/profiles"} + u.register() + + // Optionally, you can install the Swagger Service which provides a nice Web UI on your REST API + // You need to download the Swagger HTML5 assets and change the FilePath location in the config below. + // Open .appspot.com/apidocs and enter + // Place the Swagger UI files into a folder called static/swagger if you wish to use Swagger + // http://.appspot.com/apidocs.json in the api input field. + // For testing, you can use http://localhost:8080/apidocs.json + config := swagger.Config{ + // You control what services are visible + WebServices: restful.RegisteredWebServices(), + WebServicesUrl: gaeUrl(), + ApiPath: "/apidocs.json", + + // Optionally, specify where the UI is located + SwaggerPath: "/apidocs/", + + // GAE support static content which is configured in your app.yaml. + // This example expect the swagger-ui in static/swagger so you should place it there :) + SwaggerFilePath: "static/swagger"} + swagger.InstallSwaggerService(config) +} + +func (u ProfileApi) register() { + ws := new(restful.WebService) + + ws. + Path(u.Path). + // You can specify consumes and produces per route as well. + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON, restful.MIME_XML) + + ws.Route(ws.POST("").To(u.insert). + // Swagger documentation. + Doc("insert a new profile"). + Param(ws.BodyParameter("Profile", "representation of a profile").DataType("main.Profile")). + Reads(Profile{})) + + ws.Route(ws.GET("/{profile-id}").To(u.read). + // Swagger documentation. + Doc("read a profile"). + Param(ws.PathParameter("profile-id", "identifier for a profile").DataType("string")). + Writes(Profile{})) + + ws.Route(ws.PUT("/{profile-id}").To(u.update). + // Swagger documentation. + Doc("update an existing profile"). + Param(ws.PathParameter("profile-id", "identifier for a profile").DataType("string")). + Param(ws.BodyParameter("Profile", "representation of a profile").DataType("main.Profile")). + Reads(Profile{})) + + ws.Route(ws.DELETE("/{profile-id}").To(u.remove). + // Swagger documentation. + Doc("remove a profile"). + Param(ws.PathParameter("profile-id", "identifier for a profile").DataType("string"))) + + restful.Add(ws) +} + +// POST http://localhost:8080/profiles +// {"first_name": "Ivan", "nick_name": "Socks", "last_name": "Hawkes"} +// +func (u *ProfileApi) insert(r *restful.Request, w *restful.Response) { + c := appengine.NewContext(r.Request) + + // Marshall the entity from the request into a struct. + p := new(Profile) + err := r.ReadEntity(&p) + if err != nil { + w.WriteError(http.StatusNotAcceptable, err) + return + } + + // Ensure we start with a sensible value for this field. + p.LastModified = time.Now() + + // The profile belongs to this user. + p.Email = user.Current(c).String() + + k, err := datastore.Put(c, datastore.NewIncompleteKey(c, "profiles", nil), p) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Let them know the location of the newly created resource. + // TODO: Use a safe Url path append function. + w.AddHeader("Location", u.Path+"/"+k.Encode()) + + // Return the resultant entity. + w.WriteHeader(http.StatusCreated) + w.WriteEntity(p) +} + +// GET http://localhost:8080/profiles/ahdkZXZ-ZmVkZXJhdGlvbi1zZXJ2aWNlc3IVCxIIcHJvZmlsZXMYgICAgICAgAoM +// +func (u ProfileApi) read(r *restful.Request, w *restful.Response) { + c := appengine.NewContext(r.Request) + + // Decode the request parameter to determine the key for the entity. + k, err := datastore.DecodeKey(r.PathParameter("profile-id")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Retrieve the entity from the datastore. + p := Profile{} + if err := datastore.Get(c, k, &p); err != nil { + if err.Error() == "datastore: no such entity" { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Check we own the profile before allowing them to view it. + // Optionally, return a 404 instead to help prevent guessing ids. + // TODO: Allow admins access. + if p.Email != user.Current(c).String() { + http.Error(w, "You do not have access to this resource", http.StatusForbidden) + return + } + + w.WriteEntity(p) +} + +// PUT http://localhost:8080/profiles/ahdkZXZ-ZmVkZXJhdGlvbi1zZXJ2aWNlc3IVCxIIcHJvZmlsZXMYgICAgICAgAoM +// {"first_name": "Ivan", "nick_name": "Socks", "last_name": "Hawkes"} +// +func (u *ProfileApi) update(r *restful.Request, w *restful.Response) { + c := appengine.NewContext(r.Request) + + // Decode the request parameter to determine the key for the entity. + k, err := datastore.DecodeKey(r.PathParameter("profile-id")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Marshall the entity from the request into a struct. + p := new(Profile) + err = r.ReadEntity(&p) + if err != nil { + w.WriteError(http.StatusNotAcceptable, err) + return + } + + // Retrieve the old entity from the datastore. + old := Profile{} + if err := datastore.Get(c, k, &old); err != nil { + if err.Error() == "datastore: no such entity" { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Check we own the profile before allowing them to update it. + // Optionally, return a 404 instead to help prevent guessing ids. + // TODO: Allow admins access. + if old.Email != user.Current(c).String() { + http.Error(w, "You do not have access to this resource", http.StatusForbidden) + return + } + + // Since the whole entity is re-written, we need to assign any invariant fields again + // e.g. the owner of the entity. + p.Email = user.Current(c).String() + + // Keep track of the last modification date. + p.LastModified = time.Now() + + // Attempt to overwrite the old entity. + _, err = datastore.Put(c, k, p) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Let them know it succeeded. + w.WriteHeader(http.StatusNoContent) +} + +// DELETE http://localhost:8080/profiles/ahdkZXZ-ZmVkZXJhdGlvbi1zZXJ2aWNlc3IVCxIIcHJvZmlsZXMYgICAgICAgAoM +// +func (u *ProfileApi) remove(r *restful.Request, w *restful.Response) { + c := appengine.NewContext(r.Request) + + // Decode the request parameter to determine the key for the entity. + k, err := datastore.DecodeKey(r.PathParameter("profile-id")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Retrieve the old entity from the datastore. + old := Profile{} + if err := datastore.Get(c, k, &old); err != nil { + if err.Error() == "datastore: no such entity" { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + // Check we own the profile before allowing them to delete it. + // Optionally, return a 404 instead to help prevent guessing ids. + // TODO: Allow admins access. + if old.Email != user.Current(c).String() { + http.Error(w, "You do not have access to this resource", http.StatusForbidden) + return + } + + // Delete the entity. + if err := datastore.Delete(c, k); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + + // Success notification. + w.WriteHeader(http.StatusNoContent) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/google_app_engine/restful-appstats-integration.go b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/restful-appstats-integration.go new file mode 100644 index 00000000000..a871133b025 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/restful-appstats-integration.go @@ -0,0 +1,12 @@ +package main + +import ( + "github.com/mjibson/appstats" +) + +func stats(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + c := appstats.NewContext(req.Request) + chain.ProcessFilter(req, resp) + c.Stats.Status = resp.StatusCode() + c.Save() +} diff --git a/vendor/github.com/emicklei/go-restful/examples/google_app_engine/restful-user-service.go b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/restful-user-service.go new file mode 100644 index 00000000000..e1b462c3e97 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/google_app_engine/restful-user-service.go @@ -0,0 +1,162 @@ +package main + +import ( + "net/http" + + "github.com/emicklei/go-restful" + "github.com/emicklei/go-restful-swagger12" + "google.golang.org/appengine" + "google.golang.org/appengine/memcache" +) + +// This example is functionally the same as ../restful-user-service.go +// but it`s supposed to run on Goole App Engine (GAE) +// +// contributed by ivanhawkes + +type User struct { + Id, Name string +} + +type UserService struct { + // normally one would use DAO (data access object) + // but in this example we simple use memcache. +} + +func (u UserService) Register() { + ws := new(restful.WebService) + + ws. + Path("/users"). + Consumes(restful.MIME_XML, restful.MIME_JSON). + Produces(restful.MIME_JSON, restful.MIME_XML) // you can specify this per route as well + + ws.Route(ws.GET("/{user-id}").To(u.findUser). + // docs + Doc("get a user"). + Param(ws.PathParameter("user-id", "identifier of the user").DataType("string")). + Writes(User{})) // on the response + + ws.Route(ws.PATCH("").To(u.updateUser). + // docs + Doc("update a user"). + Reads(User{})) // from the request + + ws.Route(ws.PUT("/{user-id}").To(u.createUser). + // docs + Doc("create a user"). + Param(ws.PathParameter("user-id", "identifier of the user").DataType("string")). + Reads(User{})) // from the request + + ws.Route(ws.DELETE("/{user-id}").To(u.removeUser). + // docs + Doc("delete a user"). + Param(ws.PathParameter("user-id", "identifier of the user").DataType("string"))) + + restful.Add(ws) +} + +// GET http://localhost:8080/users/1 +// +func (u UserService) findUser(request *restful.Request, response *restful.Response) { + c := appengine.NewContext(request.Request) + id := request.PathParameter("user-id") + usr := new(User) + _, err := memcache.Gob.Get(c, id, &usr) + if err != nil || len(usr.Id) == 0 { + response.WriteErrorString(http.StatusNotFound, "User could not be found.") + } else { + response.WriteEntity(usr) + } +} + +// PATCH http://localhost:8080/users +// 1Melissa Raspberry +// +func (u *UserService) updateUser(request *restful.Request, response *restful.Response) { + c := appengine.NewContext(request.Request) + usr := new(User) + err := request.ReadEntity(&usr) + if err == nil { + item := &memcache.Item{ + Key: usr.Id, + Object: &usr, + } + err = memcache.Gob.Set(c, item) + if err != nil { + response.WriteError(http.StatusInternalServerError, err) + return + } + response.WriteEntity(usr) + } else { + response.WriteError(http.StatusInternalServerError, err) + } +} + +// PUT http://localhost:8080/users/1 +// 1Melissa +// +func (u *UserService) createUser(request *restful.Request, response *restful.Response) { + c := appengine.NewContext(request.Request) + usr := User{Id: request.PathParameter("user-id")} + err := request.ReadEntity(&usr) + if err == nil { + item := &memcache.Item{ + Key: usr.Id, + Object: &usr, + } + err = memcache.Gob.Add(c, item) + if err != nil { + response.WriteError(http.StatusInternalServerError, err) + return + } + response.WriteHeader(http.StatusCreated) + response.WriteEntity(usr) + } else { + response.WriteError(http.StatusInternalServerError, err) + } +} + +// DELETE http://localhost:8080/users/1 +// +func (u *UserService) removeUser(request *restful.Request, response *restful.Response) { + c := appengine.NewContext(request.Request) + id := request.PathParameter("user-id") + err := memcache.Delete(c, id) + if err != nil { + response.WriteError(http.StatusInternalServerError, err) + } +} + +func getGaeURL() string { + if appengine.IsDevAppServer() { + return "http://localhost:8080" + } else { + /** + * Include your URL on App Engine here. + * I found no way to get AppID without appengine.Context and this always + * based on a http.Request. + */ + return "http://.appspot.com" + } +} + +func init() { + u := UserService{} + u.Register() + + // Optionally, you can install the Swagger Service which provides a nice Web UI on your REST API + // You need to download the Swagger HTML5 assets and change the FilePath location in the config below. + // Open .appspot.com/apidocs and enter http://.appspot.com/apidocs.json in the api input field. + config := swagger.Config{ + WebServices: restful.RegisteredWebServices(), // you control what services are visible + WebServicesUrl: getGaeURL(), + ApiPath: "/apidocs.json", + + // Optionally, specify where the UI is located + SwaggerPath: "/apidocs/", + // GAE support static content which is configured in your app.yaml. + // This example expect the swagger-ui in static/swagger so you should place it there :) + SwaggerFilePath: "static/swagger"} + swagger.InstallSwaggerService(config) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/home.html b/vendor/github.com/emicklei/go-restful/examples/home.html new file mode 100644 index 00000000000..e5d49b42ca2 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/home.html @@ -0,0 +1,7 @@ + + + + +

{{.Text}}

+ + \ No newline at end of file diff --git a/vendor/github.com/emicklei/go-restful/examples/msgpack/msgpack_entity.go b/vendor/github.com/emicklei/go-restful/examples/msgpack/msgpack_entity.go new file mode 100644 index 00000000000..330e45896e4 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/msgpack/msgpack_entity.go @@ -0,0 +1,34 @@ +package restPack + +import ( + restful "github.com/emicklei/go-restful" + "gopkg.in/vmihailenco/msgpack.v2" +) + +const MIME_MSGPACK = "application/x-msgpack" // Accept or Content-Type used in Consumes() and/or Produces() + +// NewEntityAccessorMPack returns a new EntityReaderWriter for accessing MessagePack content. +// This package is not initialized with such an accessor using the MIME_MSGPACK contentType. +func NewEntityAccessorMsgPack() restful.EntityReaderWriter { + return entityMsgPackAccess{} +} + +// entityOctetAccess is a EntityReaderWriter for Octet encoding +type entityMsgPackAccess struct { +} + +// Read unmarshalls the value from byte slice and using msgpack to unmarshal +func (e entityMsgPackAccess) Read(req *restful.Request, v interface{}) error { + return msgpack.NewDecoder(req.Request.Body).Decode(v) +} + +// Write marshals the value to byte slice and set the Content-Type Header. +func (e entityMsgPackAccess) Write(resp *restful.Response, status int, v interface{}) error { + if v == nil { + resp.WriteHeader(status) + // do not write a nil representation + return nil + } + resp.WriteHeader(status) + return msgpack.NewEncoder(resp).Encode(v) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/msgpack/msgpack_entity_test.go b/vendor/github.com/emicklei/go-restful/examples/msgpack/msgpack_entity_test.go new file mode 100644 index 00000000000..6eb4746575e --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/msgpack/msgpack_entity_test.go @@ -0,0 +1,160 @@ +package restPack + +import ( + "bytes" + "errors" + "log" + "net/http" + "net/http/httptest" + "reflect" + "testing" + "time" + + "io/ioutil" + + restful "github.com/emicklei/go-restful" +) + +func TestMsgPack(t *testing.T) { + + // register msg pack entity + restful.RegisterEntityAccessor(MIME_MSGPACK, NewEntityAccessorMsgPack()) + type Tool struct { + Name string + Vendor string + } + + // Write + httpWriter := httptest.NewRecorder() + mpack := &Tool{Name: "json", Vendor: "apple"} + resp := restful.NewResponse(httpWriter) + resp.SetRequestAccepts("application/x-msgpack,*/*;q=0.8") + + err := resp.WriteEntity(mpack) + if err != nil { + t.Errorf("err %v", err) + } + + // Read + bodyReader := bytes.NewReader(httpWriter.Body.Bytes()) + httpRequest, _ := http.NewRequest("GET", "/test", bodyReader) + httpRequest.Header.Set("Content-Type", MIME_MSGPACK) + request := restful.NewRequest(httpRequest) + readMsgPack := new(Tool) + err = request.ReadEntity(&readMsgPack) + if err != nil { + t.Errorf("err %v", err) + } + if equal := reflect.DeepEqual(mpack, readMsgPack); !equal { + t.Fatalf("should not be error") + } +} + +func TestWithWebService(t *testing.T) { + serverURL := "http://127.0.0.1:8090" + go func() { + runRestfulMsgPackRouterServer() + }() + if err := waitForServerUp(serverURL); err != nil { + t.Errorf("%v", err) + } + + // send a post request + userData := user{Id: "0001", Name: "Tony"} + msgPackData, err := msgpack.Marshal(userData) + req, err := http.NewRequest("POST", serverURL+"/test/msgpack", bytes.NewBuffer(msgPackData)) + req.Header.Set("Content-Type", MIME_MSGPACK) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + t.Errorf("unexpected error in sending req: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Errorf("unexpected response: %v, expected: %v", resp.StatusCode, http.StatusOK) + } + + ur := &userResponse{} + expectMsgPackDocument(t, resp, ur) + if ur.Status != statusActive { + t.Fatalf("should not error") + } + log.Printf("user response:%v", ur) +} + +func expectMsgPackDocument(t *testing.T, r *http.Response, doc interface{}) { + data, err := ioutil.ReadAll(r.Body) + defer r.Body.Close() + if err != nil { + t.Errorf("ExpectMsgPackDocument: unable to read response body :%v", err) + return + } + // put the body back for re-reads + r.Body = ioutil.NopCloser(bytes.NewReader(data)) + + err = msgpack.Unmarshal(data, doc) + if err != nil { + t.Errorf("ExpectMsgPackDocument: unable to unmarshal MsgPack:%v", err) + } +} + +func runRestfulMsgPackRouterServer() { + + container := restful.NewContainer() + register(container) + + log.Print("start listening on localhost:8090") + server := &http.Server{Addr: ":8090", Handler: container} + log.Fatal(server.ListenAndServe()) +} + +func waitForServerUp(serverURL string) error { + for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) { + _, err := http.Get(serverURL + "/") + if err == nil { + return nil + } + } + return errors.New("waiting for server timed out") +} + +var ( + statusActive = "active" +) + +type user struct { + Id, Name string +} + +type userResponse struct { + Status string +} + +func register(container *restful.Container) { + restful.RegisterEntityAccessor(MIME_MSGPACK, NewEntityAccessorMsgPack()) + ws := new(restful.WebService) + ws. + Path("/test"). + Consumes(restful.MIME_JSON, MIME_MSGPACK). + Produces(restful.MIME_JSON, MIME_MSGPACK) + // route user api + ws.Route(ws.POST("/msgpack"). + To(do). + Reads(user{}). + Writes(userResponse{})) + container.Add(ws) +} + +func do(request *restful.Request, response *restful.Response) { + u := &user{} + err := request.ReadEntity(u) + if err != nil { + log.Printf("should be no error, got:%v", err) + } + log.Printf("got:%v", u) + + ur := &userResponse{Status: statusActive} + + response.SetRequestAccepts(MIME_MSGPACK) + response.WriteEntity(ur) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-CORS-filter.go b/vendor/github.com/emicklei/go-restful/examples/restful-CORS-filter.go new file mode 100644 index 00000000000..d682d43e909 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-CORS-filter.go @@ -0,0 +1,68 @@ +package main + +import ( + "io" + "log" + "net/http" + + "github.com/emicklei/go-restful" +) + +// Cross-origin resource sharing (CORS) is a mechanism that allows JavaScript on a web page +// to make XMLHttpRequests to another domain, not the domain the JavaScript originated from. +// +// http://en.wikipedia.org/wiki/Cross-origin_resource_sharing +// http://enable-cors.org/server.html +// +// GET http://localhost:8080/users +// +// GET http://localhost:8080/users/1 +// +// PUT http://localhost:8080/users/1 +// +// DELETE http://localhost:8080/users/1 +// +// OPTIONS http://localhost:8080/users/1 with Header "Origin" set to some domain and + +type UserResource struct{} + +func (u UserResource) RegisterTo(container *restful.Container) { + ws := new(restful.WebService) + ws. + Path("/users"). + Consumes("*/*"). + Produces("*/*") + + ws.Route(ws.GET("/{user-id}").To(u.nop)) + ws.Route(ws.POST("").To(u.nop)) + ws.Route(ws.PUT("/{user-id}").To(u.nop)) + ws.Route(ws.DELETE("/{user-id}").To(u.nop)) + + container.Add(ws) +} + +func (u UserResource) nop(request *restful.Request, response *restful.Response) { + io.WriteString(response.ResponseWriter, "this would be a normal response") +} + +func main() { + wsContainer := restful.NewContainer() + u := UserResource{} + u.RegisterTo(wsContainer) + + // Add container filter to enable CORS + cors := restful.CrossOriginResourceSharing{ + ExposeHeaders: []string{"X-My-Header"}, + AllowedHeaders: []string{"Content-Type", "Accept"}, + AllowedMethods: []string{"GET", "POST"}, + CookiesAllowed: false, + Container: wsContainer} + wsContainer.Filter(cors.Filter) + + // Add container filter to respond to OPTIONS + wsContainer.Filter(wsContainer.OPTIONSFilter) + + log.Print("start listening on localhost:8080") + server := &http.Server{Addr: ":8080", Handler: wsContainer} + log.Fatal(server.ListenAndServe()) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-NCSA-logging.go b/vendor/github.com/emicklei/go-restful/examples/restful-NCSA-logging.go new file mode 100644 index 00000000000..7066b96d639 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-NCSA-logging.go @@ -0,0 +1,54 @@ +package main + +import ( + "github.com/emicklei/go-restful" + "io" + "log" + "net/http" + "os" + "strings" + "time" +) + +// This example shows how to create a filter that produces log lines +// according to the Common Log Format, also known as the NCSA standard. +// +// kindly contributed by leehambley +// +// GET http://localhost:8080/ping + +var logger *log.Logger = log.New(os.Stdout, "", 0) + +func NCSACommonLogFormatLogger() restful.FilterFunction { + return func(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + var username = "-" + if req.Request.URL.User != nil { + if name := req.Request.URL.User.Username(); name != "" { + username = name + } + } + chain.ProcessFilter(req, resp) + logger.Printf("%s - %s [%s] \"%s %s %s\" %d %d", + strings.Split(req.Request.RemoteAddr, ":")[0], + username, + time.Now().Format("02/Jan/2006:15:04:05 -0700"), + req.Request.Method, + req.Request.URL.RequestURI(), + req.Request.Proto, + resp.StatusCode(), + resp.ContentLength(), + ) + } +} + +func main() { + ws := new(restful.WebService) + ws.Filter(NCSACommonLogFormatLogger()) + ws.Route(ws.GET("/ping").To(hello)) + restful.Add(ws) + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func hello(req *restful.Request, resp *restful.Response) { + io.WriteString(resp, "pong") +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-basic-authentication.go b/vendor/github.com/emicklei/go-restful/examples/restful-basic-authentication.go new file mode 100644 index 00000000000..f4fd5ce183d --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-basic-authentication.go @@ -0,0 +1,35 @@ +package main + +import ( + "github.com/emicklei/go-restful" + "io" + "log" + "net/http" +) + +// This example shows how to create a (Route) Filter that performs Basic Authentication on the Http request. +// +// GET http://localhost:8080/secret +// and use admin,admin for the credentials + +func main() { + ws := new(restful.WebService) + ws.Route(ws.GET("/secret").Filter(basicAuthenticate).To(secret)) + restful.Add(ws) + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func basicAuthenticate(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + // usr/pwd = admin/admin + u, p, ok := req.Request.BasicAuth() + if !ok || u != "admin" || p != "admin" { + resp.AddHeader("WWW-Authenticate", "Basic realm=Protected Area") + resp.WriteErrorString(401, "401: Not Authorized") + return + } + chain.ProcessFilter(req, resp) +} + +func secret(req *restful.Request, resp *restful.Response) { + io.WriteString(resp, "42") +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-cpuprofiler-service.go b/vendor/github.com/emicklei/go-restful/examples/restful-cpuprofiler-service.go new file mode 100644 index 00000000000..9148213cf01 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-cpuprofiler-service.go @@ -0,0 +1,65 @@ +package main + +import ( + "github.com/emicklei/go-restful" + "io" + "log" + "os" + "runtime/pprof" +) + +// ProfilingService is a WebService that can start/stop a CPU profile and write results to a file +// GET /{rootPath}/start will activate CPU profiling +// GET /{rootPath}/stop will stop profiling +// +// NewProfileService("/profiler", "ace.prof").AddWebServiceTo(restful.DefaultContainer) +// +type ProfilingService struct { + rootPath string // the base (root) of the service, e.g. /profiler + cpuprofile string // the output filename to write profile results, e.g. myservice.prof + cpufile *os.File // if not nil, then profiling is active +} + +func NewProfileService(rootPath string, outputFilename string) *ProfilingService { + ps := new(ProfilingService) + ps.rootPath = rootPath + ps.cpuprofile = outputFilename + return ps +} + +// Add this ProfileService to a restful Container +func (p ProfilingService) AddWebServiceTo(container *restful.Container) { + ws := new(restful.WebService) + ws.Path(p.rootPath).Consumes("*/*").Produces(restful.MIME_JSON) + ws.Route(ws.GET("/start").To(p.startProfiler)) + ws.Route(ws.GET("/stop").To(p.stopProfiler)) + container.Add(ws) +} + +func (p *ProfilingService) startProfiler(req *restful.Request, resp *restful.Response) { + if p.cpufile != nil { + io.WriteString(resp.ResponseWriter, "[restful] CPU profiling already running") + return // error? + } + cpufile, err := os.Create(p.cpuprofile) + if err != nil { + log.Fatal(err) + } + // remember for close + p.cpufile = cpufile + pprof.StartCPUProfile(cpufile) + io.WriteString(resp.ResponseWriter, "[restful] CPU profiling started, writing on:"+p.cpuprofile) +} + +func (p *ProfilingService) stopProfiler(req *restful.Request, resp *restful.Response) { + if p.cpufile == nil { + io.WriteString(resp.ResponseWriter, "[restful] CPU profiling not active") + return // error? + } + pprof.StopCPUProfile() + p.cpufile.Close() + p.cpufile = nil + io.WriteString(resp.ResponseWriter, "[restful] CPU profiling stopped, closing:"+p.cpuprofile) +} + +func main() {} // exists for example compilation only diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-curly-router.go b/vendor/github.com/emicklei/go-restful/examples/restful-curly-router.go new file mode 100644 index 00000000000..1bddb34af1f --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-curly-router.go @@ -0,0 +1,107 @@ +package main + +import ( + "log" + "net/http" + + "github.com/emicklei/go-restful" +) + +// This example has the same service definition as restful-user-resource +// but uses a different router (CurlyRouter) that does not use regular expressions +// +// POST http://localhost:8080/users +// 1Melissa Raspberry +// +// GET http://localhost:8080/users/1 +// +// PUT http://localhost:8080/users/1 +// 1Melissa +// +// DELETE http://localhost:8080/users/1 +// + +type User struct { + Id, Name string +} + +type UserResource struct { + // normally one would use DAO (data access object) + users map[string]User +} + +func (u UserResource) Register(container *restful.Container) { + ws := new(restful.WebService) + ws. + Path("/users"). + Consumes(restful.MIME_XML, restful.MIME_JSON). + Produces(restful.MIME_JSON, restful.MIME_XML) // you can specify this per route as well + + ws.Route(ws.GET("/{user-id}").To(u.findUser)) + ws.Route(ws.POST("").To(u.updateUser)) + ws.Route(ws.PUT("/{user-id}").To(u.createUser)) + ws.Route(ws.DELETE("/{user-id}").To(u.removeUser)) + + container.Add(ws) +} + +// GET http://localhost:8080/users/1 +// +func (u UserResource) findUser(request *restful.Request, response *restful.Response) { + id := request.PathParameter("user-id") + usr := u.users[id] + if len(usr.Id) == 0 { + response.AddHeader("Content-Type", "text/plain") + response.WriteErrorString(http.StatusNotFound, "User could not be found.") + } else { + response.WriteEntity(usr) + } +} + +// POST http://localhost:8080/users +// 1Melissa Raspberry +// +func (u *UserResource) updateUser(request *restful.Request, response *restful.Response) { + usr := new(User) + err := request.ReadEntity(&usr) + if err == nil { + u.users[usr.Id] = *usr + response.WriteEntity(usr) + } else { + response.AddHeader("Content-Type", "text/plain") + response.WriteErrorString(http.StatusInternalServerError, err.Error()) + } +} + +// PUT http://localhost:8080/users/1 +// 1Melissa +// +func (u *UserResource) createUser(request *restful.Request, response *restful.Response) { + usr := User{Id: request.PathParameter("user-id")} + err := request.ReadEntity(&usr) + if err == nil { + u.users[usr.Id] = usr + response.WriteHeaderAndEntity(http.StatusCreated, usr) + } else { + response.AddHeader("Content-Type", "text/plain") + response.WriteErrorString(http.StatusInternalServerError, err.Error()) + } +} + +// DELETE http://localhost:8080/users/1 +// +func (u *UserResource) removeUser(request *restful.Request, response *restful.Response) { + id := request.PathParameter("user-id") + delete(u.users, id) +} + +func main() { + wsContainer := restful.NewContainer() + wsContainer.Router(restful.CurlyRouter{}) + u := UserResource{map[string]User{}} + u.Register(wsContainer) + + log.Print("start listening on localhost:8080") + server := &http.Server{Addr: ":8080", Handler: wsContainer} + log.Fatal(server.ListenAndServe()) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-curly-router_test.go b/vendor/github.com/emicklei/go-restful/examples/restful-curly-router_test.go new file mode 100644 index 00000000000..87aed068a73 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-curly-router_test.go @@ -0,0 +1,149 @@ +package main + +import ( + "bytes" + "errors" + "log" + "net/http" + "testing" + "time" + + "github.com/emicklei/go-restful" +) + +type User struct { + Id, Name string +} + +type UserResource struct { + users map[string]User +} + +func (u UserResource) Register(container *restful.Container) { + ws := new(restful.WebService) + ws. + Path("/users"). + Consumes(restful.MIME_XML, restful.MIME_JSON). + Produces(restful.MIME_JSON, restful.MIME_XML) + + ws.Route(ws.GET("/{user-id}").To(u.findUser)) + ws.Route(ws.POST("").To(u.updateUser)) + ws.Route(ws.PUT("/{user-id}").To(u.createUser)) + ws.Route(ws.DELETE("/{user-id}").To(u.removeUser)) + + container.Add(ws) +} + +// GET http://localhost:8090/users/1 +// +func (u UserResource) findUser(request *restful.Request, response *restful.Response) { + id := request.PathParameter("user-id") + usr := u.users[id] + if len(usr.Id) == 0 { + response.AddHeader("Content-Type", "text/plain") + response.WriteErrorString(http.StatusNotFound, "User could not be found.") + } else { + response.WriteEntity(usr) + } +} + +// POST http://localhost:8090/users +// 1Melissa Raspberry +// +func (u *UserResource) updateUser(request *restful.Request, response *restful.Response) { + usr := new(User) + err := request.ReadEntity(&usr) + if err == nil { + u.users[usr.Id] = *usr + response.WriteEntity(usr) + } else { + response.AddHeader("Content-Type", "text/plain") + response.WriteErrorString(http.StatusInternalServerError, err.Error()) + } +} + +// PUT http://localhost:8090/users/1 +// 1Melissa +// +func (u *UserResource) createUser(request *restful.Request, response *restful.Response) { + usr := User{Id: request.PathParameter("user-id")} + err := request.ReadEntity(&usr) + if err == nil { + u.users[usr.Id] = usr + response.WriteHeader(http.StatusCreated) + response.WriteEntity(usr) + } else { + response.AddHeader("Content-Type", "text/plain") + response.WriteErrorString(http.StatusInternalServerError, err.Error()) + } +} + +// DELETE http://localhost:8090/users/1 +// +func (u *UserResource) removeUser(request *restful.Request, response *restful.Response) { + id := request.PathParameter("user-id") + delete(u.users, id) +} + +func RunRestfulCurlyRouterServer() { + wsContainer := restful.NewContainer() + wsContainer.Router(restful.CurlyRouter{}) + u := UserResource{map[string]User{}} + u.Register(wsContainer) + + log.Print("start listening on localhost:8090") + server := &http.Server{Addr: ":8090", Handler: wsContainer} + log.Fatal(server.ListenAndServe()) +} + +func waitForServerUp(serverURL string) error { + for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) { + _, err := http.Get(serverURL + "/") + if err == nil { + return nil + } + } + return errors.New("waiting for server timed out") +} + +func TestServer(t *testing.T) { + serverURL := "http://localhost:8090" + go func() { + RunRestfulCurlyRouterServer() + }() + if err := waitForServerUp(serverURL); err != nil { + t.Errorf("%v", err) + } + + // GET should give a 405 + resp, err := http.Get(serverURL + "/users/") + if err != nil { + t.Errorf("unexpected error in GET /users/: %v", err) + } + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Errorf("unexpected response: %v, expected: %v", resp.StatusCode, http.StatusOK) + } + + // Send a POST request. + var jsonStr = []byte(`{"id":"1","name":"user1"}`) + req, err := http.NewRequest("POST", serverURL+"/users/", bytes.NewBuffer(jsonStr)) + req.Header.Set("Content-Type", restful.MIME_JSON) + + client := &http.Client{} + resp, err = client.Do(req) + if err != nil { + t.Errorf("unexpected error in sending req: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Errorf("unexpected response: %v, expected: %v", resp.StatusCode, http.StatusOK) + } + + // Test that GET works. + resp, err = http.Get(serverURL + "/users/1") + if err != nil { + t.Errorf("unexpected error in GET /users/1: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Errorf("unexpected response: %v, expected: %v", resp.StatusCode, http.StatusOK) + } +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-encoding-filter.go b/vendor/github.com/emicklei/go-restful/examples/restful-encoding-filter.go new file mode 100644 index 00000000000..177d5a99458 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-encoding-filter.go @@ -0,0 +1,61 @@ +package main + +import ( + "github.com/emicklei/go-restful" + "log" + "net/http" +) + +type User struct { + Id, Name string +} + +type UserList struct { + Users []User +} + +// +// This example shows how to use the CompressingResponseWriter by a Filter +// such that encoding can be enabled per WebService or per Route (instead of per container) +// Using restful.DefaultContainer.EnableContentEncoding(true) will encode all responses served by WebServices in the DefaultContainer. +// +// Set Accept-Encoding to gzip or deflate +// GET http://localhost:8080/users/42 +// and look at the response headers + +func main() { + restful.Add(NewUserService()) + log.Print("start listening on localhost:8080") + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func NewUserService() *restful.WebService { + ws := new(restful.WebService) + ws. + Path("/users"). + Consumes(restful.MIME_XML, restful.MIME_JSON). + Produces(restful.MIME_JSON, restful.MIME_XML) + + // install a response encoding filter + ws.Route(ws.GET("/{user-id}").Filter(encodingFilter).To(findUser)) + return ws +} + +// Route Filter (defines FilterFunction) +func encodingFilter(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + log.Printf("[encoding-filter] %s,%s\n", req.Request.Method, req.Request.URL) + // wrap responseWriter into a compressing one + compress, _ := restful.NewCompressingResponseWriter(resp.ResponseWriter, restful.ENCODING_GZIP) + resp.ResponseWriter = compress + defer func() { + compress.Close() + }() + chain.ProcessFilter(req, resp) +} + +// GET http://localhost:8080/users/42 +// +func findUser(request *restful.Request, response *restful.Response) { + log.Print("findUser") + response.WriteEntity(User{"42", "Gandalf"}) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-filters.go b/vendor/github.com/emicklei/go-restful/examples/restful-filters.go new file mode 100644 index 00000000000..aaadb9d23bd --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-filters.go @@ -0,0 +1,114 @@ +package main + +import ( + "github.com/emicklei/go-restful" + "log" + "net/http" + "time" +) + +type User struct { + Id, Name string +} + +type UserList struct { + Users []User +} + +// This example show how to create and use the three different Filters (Container,WebService and Route) +// When applied to the restful.DefaultContainer, we refer to them as a global filter. +// +// GET http://localhost:8080/users/42 +// and see the logging per filter (try repeating this request) + +func main() { + // install a global (=DefaultContainer) filter (processed before any webservice in the DefaultContainer) + restful.Filter(globalLogging) + + restful.Add(NewUserService()) + log.Print("start listening on localhost:8080") + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func NewUserService() *restful.WebService { + ws := new(restful.WebService) + ws. + Path("/users"). + Consumes(restful.MIME_XML, restful.MIME_JSON). + Produces(restful.MIME_JSON, restful.MIME_XML) + + // install a webservice filter (processed before any route) + ws.Filter(webserviceLogging).Filter(measureTime) + + // install a counter filter + ws.Route(ws.GET("").Filter(NewCountFilter().routeCounter).To(getAllUsers)) + + // install 2 chained route filters (processed before calling findUser) + ws.Route(ws.GET("/{user-id}").Filter(routeLogging).Filter(NewCountFilter().routeCounter).To(findUser)) + return ws +} + +// Global Filter +func globalLogging(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + log.Printf("[global-filter (logger)] %s,%s\n", req.Request.Method, req.Request.URL) + chain.ProcessFilter(req, resp) +} + +// WebService Filter +func webserviceLogging(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + log.Printf("[webservice-filter (logger)] %s,%s\n", req.Request.Method, req.Request.URL) + chain.ProcessFilter(req, resp) +} + +// WebService (post-process) Filter (as a struct that defines a FilterFunction) +func measureTime(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + now := time.Now() + chain.ProcessFilter(req, resp) + log.Printf("[webservice-filter (timer)] %v\n", time.Now().Sub(now)) +} + +// Route Filter (defines FilterFunction) +func routeLogging(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + log.Printf("[route-filter (logger)] %s,%s\n", req.Request.Method, req.Request.URL) + chain.ProcessFilter(req, resp) +} + +// Route Filter (as a struct that defines a FilterFunction) +// CountFilter implements a FilterFunction for counting requests. +type CountFilter struct { + count int + counter chan int // for go-routine safe count increments +} + +// NewCountFilter creates and initializes a new CountFilter. +func NewCountFilter() *CountFilter { + c := new(CountFilter) + c.counter = make(chan int) + go func() { + for { + c.count += <-c.counter + } + }() + return c +} + +// routeCounter increments the count of the filter (through a channel) +func (c *CountFilter) routeCounter(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + c.counter <- 1 + log.Printf("[route-filter (counter)] count:%d", c.count) + chain.ProcessFilter(req, resp) +} + +// GET http://localhost:8080/users +// +func getAllUsers(request *restful.Request, response *restful.Response) { + log.Print("getAllUsers") + response.WriteEntity(UserList{[]User{{"42", "Gandalf"}, {"3.14", "Pi"}}}) +} + +// GET http://localhost:8080/users/42 +// +func findUser(request *restful.Request, response *restful.Response) { + log.Print("findUser") + response.WriteEntity(User{"42", "Gandalf"}) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-form-handling.go b/vendor/github.com/emicklei/go-restful/examples/restful-form-handling.go new file mode 100644 index 00000000000..e85608c9b09 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-form-handling.go @@ -0,0 +1,63 @@ +package main + +import ( + "fmt" + "github.com/emicklei/go-restful" + "github.com/gorilla/schema" + "io" + "log" + "net/http" +) + +// This example shows how to handle a POST of a HTML form that uses the standard x-www-form-urlencoded content-type. +// It uses the gorilla web tool kit schema package to decode the form data into a struct. +// +// GET http://localhost:8080/profiles +// + +type Profile struct { + Name string + Age int +} + +var decoder *schema.Decoder + +func main() { + decoder = schema.NewDecoder() + ws := new(restful.WebService) + ws.Route(ws.POST("/profiles").Consumes("application/x-www-form-urlencoded").To(postAdddress)) + ws.Route(ws.GET("/profiles").To(addresssForm)) + restful.Add(ws) + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func postAdddress(req *restful.Request, resp *restful.Response) { + err := req.Request.ParseForm() + if err != nil { + resp.WriteErrorString(http.StatusBadRequest, err.Error()) + return + } + p := new(Profile) + err = decoder.Decode(p, req.Request.PostForm) + if err != nil { + resp.WriteErrorString(http.StatusBadRequest, err.Error()) + return + } + io.WriteString(resp.ResponseWriter, fmt.Sprintf("Name=%s, Age=%d", p.Name, p.Age)) +} + +func addresssForm(req *restful.Request, resp *restful.Response) { + io.WriteString(resp.ResponseWriter, + ` + +

Enter Profile

+
+ + + + + +
+ + `) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-hello-world.go b/vendor/github.com/emicklei/go-restful/examples/restful-hello-world.go new file mode 100644 index 00000000000..bf987b80536 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-hello-world.go @@ -0,0 +1,23 @@ +package main + +import ( + "github.com/emicklei/go-restful" + "io" + "log" + "net/http" +) + +// This example shows the minimal code needed to get a restful.WebService working. +// +// GET http://localhost:8080/hello + +func main() { + ws := new(restful.WebService) + ws.Route(ws.GET("/hello").To(hello)) + restful.Add(ws) + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func hello(req *restful.Request, resp *restful.Response) { + io.WriteString(resp, "world") +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-html-template.go b/vendor/github.com/emicklei/go-restful/examples/restful-html-template.go new file mode 100644 index 00000000000..d76d9d1e474 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-html-template.go @@ -0,0 +1,35 @@ +package main + +import ( + "log" + "net/http" + "text/template" + + "github.com/emicklei/go-restful" +) + +// This example shows how to serve a HTML page using the standard Go template engine. +// +// GET http://localhost:8080/ + +func main() { + ws := new(restful.WebService) + ws.Route(ws.GET("/").To(home)) + restful.Add(ws) + print("open browser on http://localhost:8080/\n") + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +type Message struct { + Text string +} + +func home(req *restful.Request, resp *restful.Response) { + p := &Message{"restful-html-template demo"} + // you might want to cache compiled templates + t, err := template.ParseFiles("home.html") + if err != nil { + log.Fatalf("Template gave: %s", err) + } + t.Execute(resp.ResponseWriter, p) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-multi-containers.go b/vendor/github.com/emicklei/go-restful/examples/restful-multi-containers.go new file mode 100644 index 00000000000..3056d3ea22b --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-multi-containers.go @@ -0,0 +1,43 @@ +package main + +import ( + "github.com/emicklei/go-restful" + "io" + "log" + "net/http" +) + +// This example shows how to have a program with 2 WebServices containers +// each having a http server listening on its own port. +// +// The first "hello" is added to the restful.DefaultContainer (and uses DefaultServeMux) +// For the second "hello", a new container and ServeMux is created +// and requires a new http.Server with the container being the Handler. +// This first server is spawn in its own go-routine such that the program proceeds to create the second. +// +// GET http://localhost:8080/hello +// GET http://localhost:8081/hello + +func main() { + ws := new(restful.WebService) + ws.Route(ws.GET("/hello").To(hello)) + restful.Add(ws) + go func() { + log.Fatal(http.ListenAndServe(":8080", nil)) + }() + + container2 := restful.NewContainer() + ws2 := new(restful.WebService) + ws2.Route(ws2.GET("/hello").To(hello2)) + container2.Add(ws2) + server := &http.Server{Addr: ":8081", Handler: container2} + log.Fatal(server.ListenAndServe()) +} + +func hello(req *restful.Request, resp *restful.Response) { + io.WriteString(resp, "default world") +} + +func hello2(req *restful.Request, resp *restful.Response) { + io.WriteString(resp, "second world") +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-no-cache-filter.go b/vendor/github.com/emicklei/go-restful/examples/restful-no-cache-filter.go new file mode 100644 index 00000000000..8e4540f46f2 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-no-cache-filter.go @@ -0,0 +1,25 @@ +package main + +import ( + "io" + "log" + "net/http" + + "github.com/emicklei/go-restful" +) + +// This example shows how to use a WebService filter that passed the Http headers to disable browser cacheing. +// +// GET http://localhost:8080/hello + +func main() { + ws := new(restful.WebService) + ws.Filter(restful.NoBrowserCacheFilter) + ws.Route(ws.GET("/hello").To(hello)) + restful.Add(ws) + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func hello(req *restful.Request, resp *restful.Response) { + io.WriteString(resp, "world") +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-openapi.go b/vendor/github.com/emicklei/go-restful/examples/restful-openapi.go new file mode 100644 index 00000000000..b89e5e9b79b --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-openapi.go @@ -0,0 +1,178 @@ +package main + +// Note: this file is copied from https://github.com/emicklei/go-restful-openapi/blob/master/examples/user-resource.go + +import ( + "log" + "net/http" + + "github.com/emicklei/go-restful" + restfulspec "github.com/emicklei/go-restful-openapi" + "github.com/go-openapi/spec" +) + +type UserResource struct { + // normally one would use DAO (data access object) + users map[string]User +} + +func (u UserResource) WebService() *restful.WebService { + ws := new(restful.WebService) + ws. + Path("/users"). + Consumes(restful.MIME_XML, restful.MIME_JSON). + Produces(restful.MIME_JSON, restful.MIME_XML) // you can specify this per route as well + + tags := []string{"users"} + + ws.Route(ws.GET("/").To(u.findAllUsers). + // docs + Doc("get all users"). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes([]User{}). + Returns(200, "OK", []User{})) + + ws.Route(ws.GET("/{user-id}").To(u.findUser). + // docs + Doc("get a user"). + Param(ws.PathParameter("user-id", "identifier of the user").DataType("integer").DefaultValue("1")). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes(User{}). // on the response + Returns(200, "OK", User{}). + Returns(404, "Not Found", nil)) + + ws.Route(ws.PUT("/{user-id}").To(u.updateUser). + // docs + Doc("update a user"). + Param(ws.PathParameter("user-id", "identifier of the user").DataType("string")). + Metadata(restfulspec.KeyOpenAPITags, tags). + Reads(User{})) // from the request + + ws.Route(ws.PUT("").To(u.createUser). + // docs + Doc("create a user"). + Metadata(restfulspec.KeyOpenAPITags, tags). + Reads(User{})) // from the request + + ws.Route(ws.DELETE("/{user-id}").To(u.removeUser). + // docs + Doc("delete a user"). + Metadata(restfulspec.KeyOpenAPITags, tags). + Param(ws.PathParameter("user-id", "identifier of the user").DataType("string"))) + + return ws +} + +// GET http://localhost:8080/users +// +func (u UserResource) findAllUsers(request *restful.Request, response *restful.Response) { + list := []User{} + for _, each := range u.users { + list = append(list, each) + } + response.WriteEntity(list) +} + +// GET http://localhost:8080/users/1 +// +func (u UserResource) findUser(request *restful.Request, response *restful.Response) { + id := request.PathParameter("user-id") + usr := u.users[id] + if len(usr.ID) == 0 { + response.WriteErrorString(http.StatusNotFound, "User could not be found.") + } else { + response.WriteEntity(usr) + } +} + +// PUT http://localhost:8080/users/1 +// 1Melissa Raspberry +// +func (u *UserResource) updateUser(request *restful.Request, response *restful.Response) { + usr := new(User) + err := request.ReadEntity(&usr) + if err == nil { + u.users[usr.ID] = *usr + response.WriteEntity(usr) + } else { + response.WriteError(http.StatusInternalServerError, err) + } +} + +// PUT http://localhost:8080/users/1 +// 1Melissa +// +func (u *UserResource) createUser(request *restful.Request, response *restful.Response) { + usr := User{ID: request.PathParameter("user-id")} + err := request.ReadEntity(&usr) + if err == nil { + u.users[usr.ID] = usr + response.WriteHeaderAndEntity(http.StatusCreated, usr) + } else { + response.WriteError(http.StatusInternalServerError, err) + } +} + +// DELETE http://localhost:8080/users/1 +// +func (u *UserResource) removeUser(request *restful.Request, response *restful.Response) { + id := request.PathParameter("user-id") + delete(u.users, id) +} + +func main() { + u := UserResource{map[string]User{}} + restful.DefaultContainer.Add(u.WebService()) + + config := restfulspec.Config{ + WebServices: restful.RegisteredWebServices(), // you control what services are visible + APIPath: "/apidocs.json", + PostBuildSwaggerObjectHandler: enrichSwaggerObject} + restful.DefaultContainer.Add(restfulspec.NewOpenAPIService(config)) + + // Optionally, you can install the Swagger Service which provides a nice Web UI on your REST API + // You need to download the Swagger HTML5 assets and change the FilePath location in the config below. + // Open http://localhost:8080/apidocs/?url=http://localhost:8080/apidocs.json + http.Handle("/apidocs/", http.StripPrefix("/apidocs/", http.FileServer(http.Dir("/Users/emicklei/Projects/swagger-ui/dist")))) + + // Optionally, you may need to enable CORS for the UI to work. + cors := restful.CrossOriginResourceSharing{ + AllowedHeaders: []string{"Content-Type", "Accept"}, + AllowedMethods: []string{"GET", "POST", "PUT", "DELETE"}, + CookiesAllowed: false, + Container: restful.DefaultContainer} + restful.DefaultContainer.Filter(cors.Filter) + + log.Printf("Get the API using http://localhost:8080/apidocs.json") + log.Printf("Open Swagger UI using http://localhost:8080/apidocs/?url=http://localhost:8080/apidocs.json") + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func enrichSwaggerObject(swo *spec.Swagger) { + swo.Info = &spec.Info{ + InfoProps: spec.InfoProps{ + Title: "UserService", + Description: "Resource for managing Users", + Contact: &spec.ContactInfo{ + Name: "john", + Email: "john@doe.rp", + URL: "http://johndoe.org", + }, + License: &spec.License{ + Name: "MIT", + URL: "http://mit.org", + }, + Version: "1.0.0", + }, + } + swo.Tags = []spec.Tag{spec.Tag{TagProps: spec.TagProps{ + Name: "users", + Description: "Managing users"}}} +} + +// User is just a sample type +type User struct { + ID string `json:"id" description:"identifier of the user"` + Name string `json:"name" description:"name of the user" default:"john"` + Age int `json:"age" description:"age of the user" default:"21"` +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-options-filter.go b/vendor/github.com/emicklei/go-restful/examples/restful-options-filter.go new file mode 100644 index 00000000000..79ccce5580d --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-options-filter.go @@ -0,0 +1,51 @@ +package main + +import ( + "github.com/emicklei/go-restful" + "io" + "log" + "net/http" +) + +// This example shows how to use the OPTIONSFilter on a Container +// +// OPTIONS http://localhost:8080/users +// +// OPTIONS http://localhost:8080/users/1 + +type UserResource struct{} + +func (u UserResource) RegisterTo(container *restful.Container) { + ws := new(restful.WebService) + ws. + Path("/users"). + Consumes("*/*"). + Produces("*/*") + + ws.Route(ws.GET("/{user-id}").To(u.nop)) + ws.Route(ws.POST("").To(u.nop)) + ws.Route(ws.PUT("/{user-id}").To(u.nop)) + ws.Route(ws.DELETE("/{user-id}").To(u.nop)) + + container.Add(ws) +} + +func (u UserResource) nop(request *restful.Request, response *restful.Response) { + io.WriteString(response.ResponseWriter, "this would be a normal response") +} + +func main() { + wsContainer := restful.NewContainer() + u := UserResource{} + u.RegisterTo(wsContainer) + + // Add container filter to respond to OPTIONS + wsContainer.Filter(wsContainer.OPTIONSFilter) + + // For use on the default container, you can write + // restful.Filter(restful.OPTIONSFilter()) + + log.Print("start listening on localhost:8080") + server := &http.Server{Addr: ":8080", Handler: wsContainer} + log.Fatal(server.ListenAndServe()) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-path-tail.go b/vendor/github.com/emicklei/go-restful/examples/restful-path-tail.go new file mode 100644 index 00000000000..f30d6716ae6 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-path-tail.go @@ -0,0 +1,27 @@ +package main + +import ( + . "github.com/emicklei/go-restful" + "io" + "log" + "net/http" +) + +// This example shows how to create a Route matching the "tail" of a path. +// Requires the use of a CurlyRouter and the star "*" path parameter pattern. +// +// GET http://localhost:8080/basepath/some/other/location/test.xml + +func main() { + DefaultContainer.Router(CurlyRouter{}) + ws := new(WebService) + ws.Route(ws.GET("/basepath/{resource:*}").To(staticFromPathParam)) + Add(ws) + + println("[go-restful] serve path tails from http://localhost:8080/basepath") + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func staticFromPathParam(req *Request, resp *Response) { + io.WriteString(resp, "Tail="+req.PathParameter("resource")) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-pre-post-filters.go b/vendor/github.com/emicklei/go-restful/examples/restful-pre-post-filters.go new file mode 100644 index 00000000000..0b55f14930d --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-pre-post-filters.go @@ -0,0 +1,98 @@ +package main + +import ( + "github.com/emicklei/go-restful" + "io" + "log" + "net/http" +) + +// This example shows how the different types of filters are called in the request-response flow. +// The call chain is logged on the console when sending an http request. +// +// GET http://localhost:8080/1 +// GET http://localhost:8080/2 + +var indentLevel int + +func container_filter_A(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + log.Printf("url path:%v\n", req.Request.URL) + trace("container_filter_A: before", 1) + chain.ProcessFilter(req, resp) + trace("container_filter_A: after", -1) +} + +func container_filter_B(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + trace("container_filter_B: before", 1) + chain.ProcessFilter(req, resp) + trace("container_filter_B: after", -1) +} + +func service_filter_A(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + trace("service_filter_A: before", 1) + chain.ProcessFilter(req, resp) + trace("service_filter_A: after", -1) +} + +func service_filter_B(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + trace("service_filter_B: before", 1) + chain.ProcessFilter(req, resp) + trace("service_filter_B: after", -1) +} + +func route_filter_A(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + trace("route_filter_A: before", 1) + chain.ProcessFilter(req, resp) + trace("route_filter_A: after", -1) +} + +func route_filter_B(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { + trace("route_filter_B: before", 1) + chain.ProcessFilter(req, resp) + trace("route_filter_B: after", -1) +} + +func trace(what string, delta int) { + indented := what + if delta < 0 { + indentLevel += delta + } + for t := 0; t < indentLevel; t++ { + indented = "." + indented + } + log.Printf("%s", indented) + if delta > 0 { + indentLevel += delta + } +} + +func main() { + restful.Filter(container_filter_A) + restful.Filter(container_filter_B) + + ws1 := new(restful.WebService) + ws1.Path("/1") + ws1.Filter(service_filter_A) + ws1.Filter(service_filter_B) + ws1.Route(ws1.GET("").To(doit1).Filter(route_filter_A).Filter(route_filter_B)) + + ws2 := new(restful.WebService) + ws2.Path("/2") + ws2.Filter(service_filter_A) + ws2.Filter(service_filter_B) + ws2.Route(ws2.GET("").To(doit2).Filter(route_filter_A).Filter(route_filter_B)) + + restful.Add(ws1) + restful.Add(ws2) + + log.Print("go-restful example listing on http://localhost:8080/1 and http://localhost:8080/2") + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func doit1(req *restful.Request, resp *restful.Response) { + io.WriteString(resp, "nothing to see in 1") +} + +func doit2(req *restful.Request, resp *restful.Response) { + io.WriteString(resp, "nothing to see in 2") +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-resource-functions.go b/vendor/github.com/emicklei/go-restful/examples/restful-resource-functions.go new file mode 100644 index 00000000000..09e6e566373 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-resource-functions.go @@ -0,0 +1,63 @@ +package main + +import ( + "github.com/emicklei/go-restful" + "log" + "net/http" +) + +// This example shows how to use methods as RouteFunctions for WebServices. +// The ProductResource has a Register() method that creates and initializes +// a WebService to expose its methods as REST operations. +// The WebService is added to the restful.DefaultContainer. +// A ProductResource is typically created using some data access object. +// +// GET http://localhost:8080/products/1 +// POST http://localhost:8080/products +// 1The First + +type Product struct { + Id, Title string +} + +type ProductResource struct { + // typically reference a DAO (data-access-object) +} + +func (p ProductResource) getOne(req *restful.Request, resp *restful.Response) { + id := req.PathParameter("id") + log.Println("getting product with id:" + id) + resp.WriteEntity(Product{Id: id, Title: "test"}) +} + +func (p ProductResource) postOne(req *restful.Request, resp *restful.Response) { + updatedProduct := new(Product) + err := req.ReadEntity(updatedProduct) + if err != nil { // bad request + resp.WriteErrorString(http.StatusBadRequest, err.Error()) + return + } + log.Println("updating product with id:" + updatedProduct.Id) +} + +func (p ProductResource) Register() { + ws := new(restful.WebService) + ws.Path("/products") + ws.Consumes(restful.MIME_XML) + ws.Produces(restful.MIME_XML) + + ws.Route(ws.GET("/{id}").To(p.getOne). + Doc("get the product by its id"). + Param(ws.PathParameter("id", "identifier of the product").DataType("string"))) + + ws.Route(ws.POST("").To(p.postOne). + Doc("update or create a product"). + Param(ws.BodyParameter("Product", "a Product (XML)").DataType("main.Product"))) + + restful.Add(ws) +} + +func main() { + ProductResource{}.Register() + log.Fatal(http.ListenAndServe(":8080", nil)) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-route_test.go b/vendor/github.com/emicklei/go-restful/examples/restful-route_test.go new file mode 100644 index 00000000000..20c366bf919 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-route_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/emicklei/go-restful" +) + +var ( + Result string +) + +func TestRouteExtractParameter(t *testing.T) { + // setup service + ws := new(restful.WebService) + ws.Consumes(restful.MIME_XML) + ws.Route(ws.GET("/test/{param}").To(DummyHandler)) + restful.Add(ws) + + // setup request + writer + bodyReader := strings.NewReader("42") + httpRequest, _ := http.NewRequest("GET", "/test/THIS", bodyReader) + httpRequest.Header.Set("Content-Type", restful.MIME_XML) + httpWriter := httptest.NewRecorder() + + // run + restful.DefaultContainer.ServeHTTP(httpWriter, httpRequest) + + if Result != "THIS" { + t.Fatalf("Result is actually: %s", Result) + } +} + +func DummyHandler(rq *restful.Request, rp *restful.Response) { + Result = rq.PathParameter("param") +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-routefunction_test.go b/vendor/github.com/emicklei/go-restful/examples/restful-routefunction_test.go new file mode 100644 index 00000000000..957c0555078 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-routefunction_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/emicklei/go-restful" +) + +// This example show how to test one particular RouteFunction (getIt) +// It uses the httptest.ResponseRecorder to capture output + +func getIt(req *restful.Request, resp *restful.Response) { + resp.WriteHeader(204) +} + +func TestCallFunction(t *testing.T) { + httpReq, _ := http.NewRequest("GET", "/", nil) + req := restful.NewRequest(httpReq) + + recorder := new(httptest.ResponseRecorder) + resp := restful.NewResponse(recorder) + + getIt(req, resp) + if recorder.Code != 204 { + t.Fatalf("Missing or wrong status code:%d", recorder.Code) + } +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-serve-static.go b/vendor/github.com/emicklei/go-restful/examples/restful-serve-static.go new file mode 100644 index 00000000000..34faf607823 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-serve-static.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "net/http" + "path" + + "github.com/emicklei/go-restful" +) + +// This example shows how to define methods that serve static files +// It uses the standard http.ServeFile method +// +// GET http://localhost:8080/static/test.xml +// GET http://localhost:8080/static/ +// +// GET http://localhost:8080/static?resource=subdir/test.xml + +var rootdir = "/tmp" + +func main() { + restful.DefaultContainer.Router(restful.CurlyRouter{}) + + ws := new(restful.WebService) + ws.Route(ws.GET("/static/{subpath:*}").To(staticFromPathParam)) + ws.Route(ws.GET("/static").To(staticFromQueryParam)) + restful.Add(ws) + + println("[go-restful] serving files on http://localhost:8080/static from local /tmp") + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func staticFromPathParam(req *restful.Request, resp *restful.Response) { + actual := path.Join(rootdir, req.PathParameter("subpath")) + fmt.Printf("serving %s ... (from %s)\n", actual, req.PathParameter("subpath")) + http.ServeFile( + resp.ResponseWriter, + req.Request, + actual) +} + +func staticFromQueryParam(req *restful.Request, resp *restful.Response) { + http.ServeFile( + resp.ResponseWriter, + req.Request, + path.Join(rootdir, req.QueryParameter("resource"))) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-swagger.go b/vendor/github.com/emicklei/go-restful/examples/restful-swagger.go new file mode 100644 index 00000000000..ecbd71b200a --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-swagger.go @@ -0,0 +1,61 @@ +package main + +import ( + "log" + "net/http" + + "github.com/emicklei/go-restful" + "github.com/emicklei/go-restful-swagger12" +) + +type Book struct { + Title string + Author string +} + +func main() { + ws := new(restful.WebService) + ws.Path("/books") + ws.Consumes(restful.MIME_JSON, restful.MIME_XML) + ws.Produces(restful.MIME_JSON, restful.MIME_XML) + restful.Add(ws) + + ws.Route(ws.GET("/{medium}").To(noop). + Doc("Search all books"). + Param(ws.PathParameter("medium", "digital or paperback").DataType("string")). + Param(ws.QueryParameter("language", "en,nl,de").DataType("string")). + Param(ws.HeaderParameter("If-Modified-Since", "last known timestamp").DataType("datetime")). + Do(returns200, returns500)) + + ws.Route(ws.PUT("/{medium}").To(noop). + Doc("Add a new book"). + Param(ws.PathParameter("medium", "digital or paperback").DataType("string")). + Reads(Book{})) + + // You can install the Swagger Service which provides a nice Web UI on your REST API + // You need to download the Swagger HTML5 assets and change the FilePath location in the config below. + // Open http://localhost:8080/apidocs and enter http://localhost:8080/apidocs.json in the api input field. + config := swagger.Config{ + WebServices: restful.DefaultContainer.RegisteredWebServices(), // you control what services are visible + WebServicesUrl: "http://localhost:8080", + ApiPath: "/apidocs.json", + + // Optionally, specify where the UI is located + SwaggerPath: "/apidocs/", + SwaggerFilePath: "/Users/emicklei/xProjects/swagger-ui/dist"} + swagger.RegisterSwaggerService(config, restful.DefaultContainer) + + log.Print("start listening on localhost:8080") + server := &http.Server{Addr: ":8080", Handler: restful.DefaultContainer} + log.Fatal(server.ListenAndServe()) +} + +func noop(req *restful.Request, resp *restful.Response) {} + +func returns200(b *restful.RouteBuilder) { + b.Returns(http.StatusOK, "OK", Book{}) +} + +func returns500(b *restful.RouteBuilder) { + b.Returns(http.StatusInternalServerError, "Bummer, something went wrong", nil) +} diff --git a/vendor/github.com/emicklei/go-restful/examples/restful-user-resource.go b/vendor/github.com/emicklei/go-restful/examples/restful-user-resource.go new file mode 100644 index 00000000000..f8d8c4c6815 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/examples/restful-user-resource.go @@ -0,0 +1,169 @@ +package main + +import ( + "log" + "net/http" + + "github.com/emicklei/go-restful" + restfulspec "github.com/emicklei/go-restful-openapi" + "github.com/go-openapi/spec" +) + +// UserResource is the REST layer to the User domain +type UserResource struct { + // normally one would use DAO (data access object) + users map[string]User +} + +// WebService creates a new service that can handle REST requests for User resources. +func (u UserResource) WebService() *restful.WebService { + ws := new(restful.WebService) + ws. + Path("/users"). + Consumes(restful.MIME_XML, restful.MIME_JSON). + Produces(restful.MIME_JSON, restful.MIME_XML) // you can specify this per route as well + + tags := []string{"users"} + + ws.Route(ws.GET("/").To(u.findAllUsers). + // docs + Doc("get all users"). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes([]User{}). + Returns(200, "OK", []User{})) + + ws.Route(ws.GET("/{user-id}").To(u.findUser). + // docs + Doc("get a user"). + Param(ws.PathParameter("user-id", "identifier of the user").DataType("integer").DefaultValue("1")). + Metadata(restfulspec.KeyOpenAPITags, tags). + Writes(User{}). // on the response + Returns(200, "OK", User{}). + Returns(404, "Not Found", nil)) + + ws.Route(ws.PUT("/{user-id}").To(u.updateUser). + // docs + Doc("update a user"). + Param(ws.PathParameter("user-id", "identifier of the user").DataType("string")). + Metadata(restfulspec.KeyOpenAPITags, tags). + Reads(User{})) // from the request + + ws.Route(ws.PUT("").To(u.createUser). + // docs + Doc("create a user"). + Metadata(restfulspec.KeyOpenAPITags, tags). + Reads(User{})) // from the request + + ws.Route(ws.DELETE("/{user-id}").To(u.removeUser). + // docs + Doc("delete a user"). + Metadata(restfulspec.KeyOpenAPITags, tags). + Param(ws.PathParameter("user-id", "identifier of the user").DataType("string"))) + + return ws +} + +// GET http://localhost:8080/users +// +func (u UserResource) findAllUsers(request *restful.Request, response *restful.Response) { + list := []User{} + for _, each := range u.users { + list = append(list, each) + } + response.WriteEntity(list) +} + +// GET http://localhost:8080/users/1 +// +func (u UserResource) findUser(request *restful.Request, response *restful.Response) { + id := request.PathParameter("user-id") + usr := u.users[id] + if len(usr.ID) == 0 { + response.WriteErrorString(http.StatusNotFound, "User could not be found.") + } else { + response.WriteEntity(usr) + } +} + +// PUT http://localhost:8080/users/1 +// 1Melissa Raspberry +// +func (u *UserResource) updateUser(request *restful.Request, response *restful.Response) { + usr := new(User) + err := request.ReadEntity(&usr) + if err == nil { + u.users[usr.ID] = *usr + response.WriteEntity(usr) + } else { + response.WriteError(http.StatusInternalServerError, err) + } +} + +// PUT http://localhost:8080/users/1 +// 1Melissa +// +func (u *UserResource) createUser(request *restful.Request, response *restful.Response) { + usr := User{ID: request.PathParameter("user-id")} + err := request.ReadEntity(&usr) + if err == nil { + u.users[usr.ID] = usr + response.WriteHeaderAndEntity(http.StatusCreated, usr) + } else { + response.WriteError(http.StatusInternalServerError, err) + } +} + +// DELETE http://localhost:8080/users/1 +// +func (u *UserResource) removeUser(request *restful.Request, response *restful.Response) { + id := request.PathParameter("user-id") + delete(u.users, id) +} + +func main() { + u := UserResource{map[string]User{}} + restful.DefaultContainer.Add(u.WebService()) + + config := restfulspec.Config{ + WebServices: restful.RegisteredWebServices(), // you control what services are visible + APIPath: "/apidocs.json", + PostBuildSwaggerObjectHandler: enrichSwaggerObject} + restful.DefaultContainer.Add(restfulspec.NewOpenAPIService(config)) + + // Optionally, you can install the Swagger Service which provides a nice Web UI on your REST API + // You need to download the Swagger HTML5 assets and change the FilePath location in the config below. + // Open http://localhost:8080/apidocs/?url=http://localhost:8080/apidocs.json + http.Handle("/apidocs/", http.StripPrefix("/apidocs/", http.FileServer(http.Dir("/Users/emicklei/Projects/swagger-ui/dist")))) + + log.Printf("start listening on localhost:8080") + log.Fatal(http.ListenAndServe(":8080", nil)) +} + +func enrichSwaggerObject(swo *spec.Swagger) { + swo.Info = &spec.Info{ + InfoProps: spec.InfoProps{ + Title: "UserService", + Description: "Resource for managing Users", + Contact: &spec.ContactInfo{ + Name: "john", + Email: "john@doe.rp", + URL: "http://johndoe.org", + }, + License: &spec.License{ + Name: "MIT", + URL: "http://mit.org", + }, + Version: "1.0.0", + }, + } + swo.Tags = []spec.Tag{spec.Tag{TagProps: spec.TagProps{ + Name: "users", + Description: "Managing users"}}} +} + +// User is just a sample type +type User struct { + ID string `json:"id" description:"identifier of the user"` + Name string `json:"name" description:"name of the user" default:"john"` + Age int `json:"age" description:"age of the user" default:"21"` +} diff --git a/vendor/github.com/emicklei/go-restful/filter.go b/vendor/github.com/emicklei/go-restful/filter.go new file mode 100644 index 00000000000..c23bfb591ad --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/filter.go @@ -0,0 +1,35 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +// FilterChain is a request scoped object to process one or more filters before calling the target RouteFunction. +type FilterChain struct { + Filters []FilterFunction // ordered list of FilterFunction + Index int // index into filters that is currently in progress + Target RouteFunction // function to call after passing all filters +} + +// ProcessFilter passes the request,response pair through the next of Filters. +// Each filter can decide to proceed to the next Filter or handle the Response itself. +func (f *FilterChain) ProcessFilter(request *Request, response *Response) { + if f.Index < len(f.Filters) { + f.Index++ + f.Filters[f.Index-1](request, response, f) + } else { + f.Target(request, response) + } +} + +// FilterFunction definitions must call ProcessFilter on the FilterChain to pass on the control and eventually call the RouteFunction +type FilterFunction func(*Request, *Response, *FilterChain) + +// NoBrowserCacheFilter is a filter function to set HTTP headers that disable browser caching +// See examples/restful-no-cache-filter.go for usage +func NoBrowserCacheFilter(req *Request, resp *Response, chain *FilterChain) { + resp.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") // HTTP 1.1. + resp.Header().Set("Pragma", "no-cache") // HTTP 1.0. + resp.Header().Set("Expires", "0") // Proxies. + chain.ProcessFilter(req, resp) +} diff --git a/vendor/github.com/emicklei/go-restful/filter_test.go b/vendor/github.com/emicklei/go-restful/filter_test.go new file mode 100644 index 00000000000..fadfb570f6c --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/filter_test.go @@ -0,0 +1,141 @@ +package restful + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" +) + +func setupServices(addGlobalFilter bool, addServiceFilter bool, addRouteFilter bool) { + if addGlobalFilter { + Filter(globalFilter) + } + Add(newTestService(addServiceFilter, addRouteFilter)) +} + +func tearDown() { + DefaultContainer.webServices = []*WebService{} + DefaultContainer.isRegisteredOnRoot = true // this allows for setupServices multiple times + DefaultContainer.containerFilters = []FilterFunction{} +} + +func newTestService(addServiceFilter bool, addRouteFilter bool) *WebService { + ws := new(WebService).Path("") + if addServiceFilter { + ws.Filter(serviceFilter) + } + rb := ws.GET("/foo").To(foo) + if addRouteFilter { + rb.Filter(routeFilter) + } + ws.Route(rb) + ws.Route(ws.GET("/bar").To(bar)) + return ws +} + +func foo(req *Request, resp *Response) { + io.WriteString(resp.ResponseWriter, "foo") +} + +func bar(req *Request, resp *Response) { + io.WriteString(resp.ResponseWriter, "bar") +} + +func fail(req *Request, resp *Response) { + http.Error(resp.ResponseWriter, "something failed", http.StatusInternalServerError) +} + +func globalFilter(req *Request, resp *Response, chain *FilterChain) { + io.WriteString(resp.ResponseWriter, "global-") + chain.ProcessFilter(req, resp) +} + +func serviceFilter(req *Request, resp *Response, chain *FilterChain) { + io.WriteString(resp.ResponseWriter, "service-") + chain.ProcessFilter(req, resp) +} + +func routeFilter(req *Request, resp *Response, chain *FilterChain) { + io.WriteString(resp.ResponseWriter, "route-") + chain.ProcessFilter(req, resp) +} + +func TestNoFilter(t *testing.T) { + tearDown() + setupServices(false, false, false) + actual := sendIt("http://example.com/foo") + if "foo" != actual { + t.Fatal("expected: foo but got:" + actual) + } +} + +func TestGlobalFilter(t *testing.T) { + tearDown() + setupServices(true, false, false) + actual := sendIt("http://example.com/foo") + if "global-foo" != actual { + t.Fatal("expected: global-foo but got:" + actual) + } +} + +func TestWebServiceFilter(t *testing.T) { + tearDown() + setupServices(true, true, false) + actual := sendIt("http://example.com/foo") + if "global-service-foo" != actual { + t.Fatal("expected: global-service-foo but got:" + actual) + } +} + +func TestRouteFilter(t *testing.T) { + tearDown() + setupServices(true, true, true) + actual := sendIt("http://example.com/foo") + if "global-service-route-foo" != actual { + t.Fatal("expected: global-service-route-foo but got:" + actual) + } +} + +func TestRouteFilterOnly(t *testing.T) { + tearDown() + setupServices(false, false, true) + actual := sendIt("http://example.com/foo") + if "route-foo" != actual { + t.Fatal("expected: route-foo but got:" + actual) + } +} + +func TestBar(t *testing.T) { + tearDown() + setupServices(false, true, false) + actual := sendIt("http://example.com/bar") + if "service-bar" != actual { + t.Fatal("expected: service-bar but got:" + actual) + } +} + +func TestAllFiltersBar(t *testing.T) { + tearDown() + setupServices(true, true, true) + actual := sendIt("http://example.com/bar") + if "global-service-bar" != actual { + t.Fatal("expected: global-service-bar but got:" + actual) + } +} + +func sendIt(address string) string { + httpRequest, _ := http.NewRequest("GET", address, nil) + httpRequest.Header.Set("Accept", "*/*") + httpWriter := httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + return httpWriter.Body.String() +} + +func sendItTo(address string, container *Container) string { + httpRequest, _ := http.NewRequest("GET", address, nil) + httpRequest.Header.Set("Accept", "*/*") + httpWriter := httptest.NewRecorder() + container.dispatch(httpWriter, httpRequest) + return httpWriter.Body.String() +} diff --git a/vendor/github.com/emicklei/go-restful/json.go b/vendor/github.com/emicklei/go-restful/json.go new file mode 100644 index 00000000000..871165166a1 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/json.go @@ -0,0 +1,11 @@ +// +build !jsoniter + +package restful + +import "encoding/json" + +var ( + MarshalIndent = json.MarshalIndent + NewDecoder = json.NewDecoder + NewEncoder = json.NewEncoder +) diff --git a/vendor/github.com/emicklei/go-restful/jsoniter.go b/vendor/github.com/emicklei/go-restful/jsoniter.go new file mode 100644 index 00000000000..11b8f8ae7f1 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/jsoniter.go @@ -0,0 +1,12 @@ +// +build jsoniter + +package restful + +import "github.com/json-iterator/go" + +var ( + json = jsoniter.ConfigCompatibleWithStandardLibrary + MarshalIndent = json.MarshalIndent + NewDecoder = json.NewDecoder + NewEncoder = json.NewEncoder +) diff --git a/vendor/github.com/emicklei/go-restful/jsr311.go b/vendor/github.com/emicklei/go-restful/jsr311.go new file mode 100644 index 00000000000..4360b492ec1 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/jsr311.go @@ -0,0 +1,293 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "errors" + "fmt" + "net/http" + "sort" +) + +// RouterJSR311 implements the flow for matching Requests to Routes (and consequently Resource Functions) +// as specified by the JSR311 http://jsr311.java.net/nonav/releases/1.1/spec/spec.html. +// RouterJSR311 implements the Router interface. +// Concept of locators is not implemented. +type RouterJSR311 struct{} + +// SelectRoute is part of the Router interface and returns the best match +// for the WebService and its Route for the given Request. +func (r RouterJSR311) SelectRoute( + webServices []*WebService, + httpRequest *http.Request) (selectedService *WebService, selectedRoute *Route, err error) { + + // Identify the root resource class (WebService) + dispatcher, finalMatch, err := r.detectDispatcher(httpRequest.URL.Path, webServices) + if err != nil { + return nil, nil, NewError(http.StatusNotFound, "") + } + // Obtain the set of candidate methods (Routes) + routes := r.selectRoutes(dispatcher, finalMatch) + if len(routes) == 0 { + return dispatcher, nil, NewError(http.StatusNotFound, "404: Page Not Found") + } + + // Identify the method (Route) that will handle the request + route, ok := r.detectRoute(routes, httpRequest) + return dispatcher, route, ok +} + +// ExtractParameters is used to obtain the path parameters from the route using the same matching +// engine as the JSR 311 router. +func (r RouterJSR311) ExtractParameters(route *Route, webService *WebService, urlPath string) map[string]string { + webServiceExpr := webService.pathExpr + webServiceMatches := webServiceExpr.Matcher.FindStringSubmatch(urlPath) + pathParameters := r.extractParams(webServiceExpr, webServiceMatches) + routeExpr := route.pathExpr + routeMatches := routeExpr.Matcher.FindStringSubmatch(webServiceMatches[len(webServiceMatches)-1]) + routeParams := r.extractParams(routeExpr, routeMatches) + for key, value := range routeParams { + pathParameters[key] = value + } + return pathParameters +} + +func (RouterJSR311) extractParams(pathExpr *pathExpression, matches []string) map[string]string { + params := map[string]string{} + for i := 1; i < len(matches); i++ { + if len(pathExpr.VarNames) >= i { + params[pathExpr.VarNames[i-1]] = matches[i] + } + } + return params +} + +// http://jsr311.java.net/nonav/releases/1.1/spec/spec3.html#x3-360003.7.2 +func (r RouterJSR311) detectRoute(routes []Route, httpRequest *http.Request) (*Route, error) { + ifOk := []Route{} + for _, each := range routes { + ok := true + for _, fn := range each.If { + if !fn(httpRequest) { + ok = false + break + } + } + if ok { + ifOk = append(ifOk, each) + } + } + if len(ifOk) == 0 { + if trace { + traceLogger.Printf("no Route found (from %d) that passes conditional checks", len(routes)) + } + return nil, NewError(http.StatusNotFound, "404: Not Found") + } + + // http method + methodOk := []Route{} + for _, each := range ifOk { + if httpRequest.Method == each.Method { + methodOk = append(methodOk, each) + } + } + if len(methodOk) == 0 { + if trace { + traceLogger.Printf("no Route found (in %d routes) that matches HTTP method %s\n", len(routes), httpRequest.Method) + } + return nil, NewError(http.StatusMethodNotAllowed, "405: Method Not Allowed") + } + inputMediaOk := methodOk + + // content-type + contentType := httpRequest.Header.Get(HEADER_ContentType) + inputMediaOk = []Route{} + for _, each := range methodOk { + if each.matchesContentType(contentType) { + inputMediaOk = append(inputMediaOk, each) + } + } + if len(inputMediaOk) == 0 { + if trace { + traceLogger.Printf("no Route found (from %d) that matches HTTP Content-Type: %s\n", len(methodOk), contentType) + } + return nil, NewError(http.StatusUnsupportedMediaType, "415: Unsupported Media Type") + } + + // accept + outputMediaOk := []Route{} + accept := httpRequest.Header.Get(HEADER_Accept) + if len(accept) == 0 { + accept = "*/*" + } + for _, each := range inputMediaOk { + if each.matchesAccept(accept) { + outputMediaOk = append(outputMediaOk, each) + } + } + if len(outputMediaOk) == 0 { + if trace { + traceLogger.Printf("no Route found (from %d) that matches HTTP Accept: %s\n", len(inputMediaOk), accept) + } + return nil, NewError(http.StatusNotAcceptable, "406: Not Acceptable") + } + // return r.bestMatchByMedia(outputMediaOk, contentType, accept), nil + return &outputMediaOk[0], nil +} + +// http://jsr311.java.net/nonav/releases/1.1/spec/spec3.html#x3-360003.7.2 +// n/m > n/* > */* +func (r RouterJSR311) bestMatchByMedia(routes []Route, contentType string, accept string) *Route { + // TODO + return &routes[0] +} + +// http://jsr311.java.net/nonav/releases/1.1/spec/spec3.html#x3-360003.7.2 (step 2) +func (r RouterJSR311) selectRoutes(dispatcher *WebService, pathRemainder string) []Route { + filtered := &sortableRouteCandidates{} + for _, each := range dispatcher.Routes() { + pathExpr := each.pathExpr + matches := pathExpr.Matcher.FindStringSubmatch(pathRemainder) + if matches != nil { + lastMatch := matches[len(matches)-1] + if len(lastMatch) == 0 || lastMatch == "/" { // do not include if value is neither empty nor ‘/’. + filtered.candidates = append(filtered.candidates, + routeCandidate{each, len(matches) - 1, pathExpr.LiteralCount, pathExpr.VarCount}) + } + } + } + if len(filtered.candidates) == 0 { + if trace { + traceLogger.Printf("WebService on path %s has no routes that match URL path remainder:%s\n", dispatcher.rootPath, pathRemainder) + } + return []Route{} + } + sort.Sort(sort.Reverse(filtered)) + + // select other routes from candidates whoes expression matches rmatch + matchingRoutes := []Route{filtered.candidates[0].route} + for c := 1; c < len(filtered.candidates); c++ { + each := filtered.candidates[c] + if each.route.pathExpr.Matcher.MatchString(pathRemainder) { + matchingRoutes = append(matchingRoutes, each.route) + } + } + return matchingRoutes +} + +// http://jsr311.java.net/nonav/releases/1.1/spec/spec3.html#x3-360003.7.2 (step 1) +func (r RouterJSR311) detectDispatcher(requestPath string, dispatchers []*WebService) (*WebService, string, error) { + filtered := &sortableDispatcherCandidates{} + for _, each := range dispatchers { + matches := each.pathExpr.Matcher.FindStringSubmatch(requestPath) + if matches != nil { + filtered.candidates = append(filtered.candidates, + dispatcherCandidate{each, matches[len(matches)-1], len(matches), each.pathExpr.LiteralCount, each.pathExpr.VarCount}) + } + } + if len(filtered.candidates) == 0 { + if trace { + traceLogger.Printf("no WebService was found to match URL path:%s\n", requestPath) + } + return nil, "", errors.New("not found") + } + sort.Sort(sort.Reverse(filtered)) + return filtered.candidates[0].dispatcher, filtered.candidates[0].finalMatch, nil +} + +// Types and functions to support the sorting of Routes + +type routeCandidate struct { + route Route + matchesCount int // the number of capturing groups + literalCount int // the number of literal characters (means those not resulting from template variable substitution) + nonDefaultCount int // the number of capturing groups with non-default regular expressions (i.e. not ‘([^ /]+?)’) +} + +func (r routeCandidate) expressionToMatch() string { + return r.route.pathExpr.Source +} + +func (r routeCandidate) String() string { + return fmt.Sprintf("(m=%d,l=%d,n=%d)", r.matchesCount, r.literalCount, r.nonDefaultCount) +} + +type sortableRouteCandidates struct { + candidates []routeCandidate +} + +func (rcs *sortableRouteCandidates) Len() int { + return len(rcs.candidates) +} +func (rcs *sortableRouteCandidates) Swap(i, j int) { + rcs.candidates[i], rcs.candidates[j] = rcs.candidates[j], rcs.candidates[i] +} +func (rcs *sortableRouteCandidates) Less(i, j int) bool { + ci := rcs.candidates[i] + cj := rcs.candidates[j] + // primary key + if ci.literalCount < cj.literalCount { + return true + } + if ci.literalCount > cj.literalCount { + return false + } + // secundary key + if ci.matchesCount < cj.matchesCount { + return true + } + if ci.matchesCount > cj.matchesCount { + return false + } + // tertiary key + if ci.nonDefaultCount < cj.nonDefaultCount { + return true + } + if ci.nonDefaultCount > cj.nonDefaultCount { + return false + } + // quaternary key ("source" is interpreted as Path) + return ci.route.Path < cj.route.Path +} + +// Types and functions to support the sorting of Dispatchers + +type dispatcherCandidate struct { + dispatcher *WebService + finalMatch string + matchesCount int // the number of capturing groups + literalCount int // the number of literal characters (means those not resulting from template variable substitution) + nonDefaultCount int // the number of capturing groups with non-default regular expressions (i.e. not ‘([^ /]+?)’) +} +type sortableDispatcherCandidates struct { + candidates []dispatcherCandidate +} + +func (dc *sortableDispatcherCandidates) Len() int { + return len(dc.candidates) +} +func (dc *sortableDispatcherCandidates) Swap(i, j int) { + dc.candidates[i], dc.candidates[j] = dc.candidates[j], dc.candidates[i] +} +func (dc *sortableDispatcherCandidates) Less(i, j int) bool { + ci := dc.candidates[i] + cj := dc.candidates[j] + // primary key + if ci.matchesCount < cj.matchesCount { + return true + } + if ci.matchesCount > cj.matchesCount { + return false + } + // secundary key + if ci.literalCount < cj.literalCount { + return true + } + if ci.literalCount > cj.literalCount { + return false + } + // tertiary key + return ci.nonDefaultCount < cj.nonDefaultCount +} diff --git a/vendor/github.com/emicklei/go-restful/jsr311_test.go b/vendor/github.com/emicklei/go-restful/jsr311_test.go new file mode 100644 index 00000000000..15cb3333c45 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/jsr311_test.go @@ -0,0 +1,332 @@ +package restful + +import ( + "io" + "net/http" + "reflect" + "sort" + "testing" +) + +// +// Step 1 tests +// +var paths = []struct { + // url with path (1) is handled by service with root (2) and last capturing group has value final (3) + path, root, final string + params map[string]string +}{ + {"/", "/", "/", map[string]string{}}, + {"/p", "/p", "", map[string]string{}}, + {"/p/x", "/p/{q}", "", map[string]string{"q": "x"}}, + {"/q/x", "/q", "/x", map[string]string{}}, + {"/p/x/", "/p/{q}", "/", map[string]string{"q": "x"}}, + {"/p/x/y", "/p/{q}", "/y", map[string]string{"q": "x"}}, + {"/q/x/y", "/q", "/x/y", map[string]string{}}, + {"/z/q", "/{p}/q", "", map[string]string{"p": "z"}}, + {"/a/b/c/q", "/", "/a/b/c/q", map[string]string{}}, +} + +func TestDetectDispatcher(t *testing.T) { + ws1 := new(WebService).Path("/") + ws2 := new(WebService).Path("/p") + ws3 := new(WebService).Path("/q") + ws4 := new(WebService).Path("/p/q") + ws5 := new(WebService).Path("/p/{q}") + ws6 := new(WebService).Path("/p/{q}/") + ws7 := new(WebService).Path("/{p}/q") + var dispatchers = []*WebService{ws1, ws2, ws3, ws4, ws5, ws6, ws7} + + wc := NewContainer() + for _, each := range dispatchers { + each.Route(each.GET("").To(dummy)) + wc.Add(each) + } + + router := RouterJSR311{} + + ok := true + for i, fixture := range paths { + who, final, err := router.detectDispatcher(fixture.path, dispatchers) + if err != nil { + t.Logf("error in detection:%v", err) + ok = false + } + if who.RootPath() != fixture.root { + t.Logf("[line:%v] Unexpected dispatcher, expected:%v, actual:%v", i, fixture.root, who.RootPath()) + ok = false + } + if final != fixture.final { + t.Logf("[line:%v] Unexpected final, expected:%v, actual:%v", i, fixture.final, final) + ok = false + } + params := router.ExtractParameters(&who.Routes()[0], who, fixture.path) + if !reflect.DeepEqual(params, fixture.params) { + t.Logf("[line:%v] Unexpected params, expected:%v, actual:%v", i, fixture.params, params) + ok = false + } + } + if !ok { + t.Fail() + } +} + +// +// Step 2 tests +// + +// go test -v -test.run TestISSUE_179 ...restful +func TestISSUE_179(t *testing.T) { + ws1 := new(WebService) + ws1.Route(ws1.GET("/v1/category/{param:*}").To(dummy)) + routes := RouterJSR311{}.selectRoutes(ws1, "/v1/category/sub/sub") + t.Logf("%v", routes) +} + +// go test -v -test.run TestISSUE_30 ...restful +func TestISSUE_30(t *testing.T) { + ws1 := new(WebService).Path("/users") + ws1.Route(ws1.GET("/{id}").To(dummy)) + ws1.Route(ws1.POST("/login").To(dummy)) + routes := RouterJSR311{}.selectRoutes(ws1, "/login") + if len(routes) != 2 { + t.Fatal("expected 2 routes") + } + if routes[0].Path != "/users/login" { + t.Error("first is", routes[0].Path) + t.Logf("routes:%v", routes) + } +} + +// go test -v -test.run TestISSUE_34 ...restful +func TestISSUE_34(t *testing.T) { + ws1 := new(WebService).Path("/") + ws1.Route(ws1.GET("/{type}/{id}").To(dummy)) + ws1.Route(ws1.GET("/network/{id}").To(dummy)) + routes := RouterJSR311{}.selectRoutes(ws1, "/network/12") + if len(routes) != 2 { + t.Fatal("expected 2 routes") + } + if routes[0].Path != "/network/{id}" { + t.Error("first is", routes[0].Path) + t.Logf("routes:%v", routes) + } +} + +// go test -v -test.run TestISSUE_34_2 ...restful +func TestISSUE_34_2(t *testing.T) { + ws1 := new(WebService).Path("/") + // change the registration order + ws1.Route(ws1.GET("/network/{id}").To(dummy)) + ws1.Route(ws1.GET("/{type}/{id}").To(dummy)) + routes := RouterJSR311{}.selectRoutes(ws1, "/network/12") + if len(routes) != 2 { + t.Fatal("expected 2 routes") + } + if routes[0].Path != "/network/{id}" { + t.Error("first is", routes[0].Path) + } +} + +// go test -v -test.run TestISSUE_137 ...restful +func TestISSUE_137(t *testing.T) { + ws1 := new(WebService) + ws1.Route(ws1.GET("/hello").To(dummy)) + routes := RouterJSR311{}.selectRoutes(ws1, "/") + t.Log(routes) + if len(routes) > 0 { + t.Error("no route expected") + } +} + +func TestSelectRoutesSlash(t *testing.T) { + ws1 := new(WebService).Path("/") + ws1.Route(ws1.GET("").To(dummy)) + ws1.Route(ws1.GET("/").To(dummy)) + ws1.Route(ws1.GET("/u").To(dummy)) + ws1.Route(ws1.POST("/u").To(dummy)) + ws1.Route(ws1.POST("/u/v").To(dummy)) + ws1.Route(ws1.POST("/u/{w}").To(dummy)) + ws1.Route(ws1.POST("/u/{w}/z").To(dummy)) + routes := RouterJSR311{}.selectRoutes(ws1, "/u") + checkRoutesContains(routes, "/u", t) + checkRoutesContainsNo(routes, "/u/v", t) + checkRoutesContainsNo(routes, "/", t) + checkRoutesContainsNo(routes, "/u/{w}/z", t) +} +func TestSelectRoutesU(t *testing.T) { + ws1 := new(WebService).Path("/u") + ws1.Route(ws1.GET("").To(dummy)) + ws1.Route(ws1.GET("/").To(dummy)) + ws1.Route(ws1.GET("/v").To(dummy)) + ws1.Route(ws1.POST("/{w}").To(dummy)) + ws1.Route(ws1.POST("/{w}/z").To(dummy)) // so full path = /u/{w}/z + routes := RouterJSR311{}.selectRoutes(ws1, "/v") // test against /u/v + checkRoutesContains(routes, "/u/{w}", t) +} + +func TestSelectRoutesUsers1(t *testing.T) { + ws1 := new(WebService).Path("/users") + ws1.Route(ws1.POST("").To(dummy)) + ws1.Route(ws1.POST("/").To(dummy)) + ws1.Route(ws1.PUT("/{id}").To(dummy)) + routes := RouterJSR311{}.selectRoutes(ws1, "/1") + checkRoutesContains(routes, "/users/{id}", t) +} +func checkRoutesContains(routes []Route, path string, t *testing.T) { + if !containsRoutePath(routes, path, t) { + for _, r := range routes { + t.Logf("route %v %v", r.Method, r.Path) + } + t.Fatalf("routes should include [%v]:", path) + } +} +func checkRoutesContainsNo(routes []Route, path string, t *testing.T) { + if containsRoutePath(routes, path, t) { + for _, r := range routes { + t.Logf("route %v %v", r.Method, r.Path) + } + t.Fatalf("routes should not include [%v]:", path) + } +} +func containsRoutePath(routes []Route, path string, t *testing.T) bool { + for _, each := range routes { + if each.Path == path { + return true + } + } + return false +} + +// go test -v -test.run TestSortableRouteCandidates ...restful +func TestSortableRouteCandidates(t *testing.T) { + fixture := &sortableRouteCandidates{} + r1 := routeCandidate{matchesCount: 0, literalCount: 0, nonDefaultCount: 0} + r2 := routeCandidate{matchesCount: 0, literalCount: 0, nonDefaultCount: 1} + r3 := routeCandidate{matchesCount: 0, literalCount: 1, nonDefaultCount: 1} + r4 := routeCandidate{matchesCount: 1, literalCount: 1, nonDefaultCount: 0} + r5 := routeCandidate{matchesCount: 1, literalCount: 0, nonDefaultCount: 0} + fixture.candidates = append(fixture.candidates, r5, r4, r3, r2, r1) + sort.Sort(sort.Reverse(fixture)) + first := fixture.candidates[0] + if first.matchesCount != 1 && first.literalCount != 1 && first.nonDefaultCount != 0 { + t.Fatal("expected r4") + } + last := fixture.candidates[len(fixture.candidates)-1] + if last.matchesCount != 0 && last.literalCount != 0 && last.nonDefaultCount != 0 { + t.Fatal("expected r1") + } +} + +func TestDetectRouteReturns404IfNoRoutePassesConditions(t *testing.T) { + called := false + shouldNotBeCalledButWas := false + + routes := []Route{ + new(RouteBuilder).To(dummy). + If(func(req *http.Request) bool { return false }). + Build(), + + // check that condition functions are called in order + new(RouteBuilder). + To(dummy). + If(func(req *http.Request) bool { return true }). + If(func(req *http.Request) bool { called = true; return false }). + Build(), + + // check that condition functions short circuit + new(RouteBuilder). + To(dummy). + If(func(req *http.Request) bool { return false }). + If(func(req *http.Request) bool { shouldNotBeCalledButWas = true; return false }). + Build(), + } + + _, err := RouterJSR311{}.detectRoute(routes, (*http.Request)(nil)) + if se := err.(ServiceError); se.Code != 404 { + t.Fatalf("expected 404, got %d", se.Code) + } + + if !called { + t.Fatal("expected condition function to get called, but it wasn't") + } + + if shouldNotBeCalledButWas { + t.Fatal("expected condition function to not be called, but it was") + } +} + +var extractParams = []struct { + name string + routePath string + urlPath string + expectedParams map[string]string +}{ + {"wildcardLastPart", "/fixed/{var:*}", "/fixed/remainder", map[string]string{"var": "remainder"}}, + {"wildcardMultipleParts", "/fixed/{var:*}", "/fixed/remain/der", map[string]string{"var": "remain/der"}}, + {"wildcardManyParts", "/fixed/{var:*}", "/fixed/test/sub/hi.html", map[string]string{"var": "test/sub/hi.html"}}, + {"wildcardInMiddle", "/fixed/{var:*}/morefixed", "/fixed/middle/stuff/morefixed", map[string]string{"var": "middle/stuff"}}, + {"wildcardFollowedByVar", "/fixed/{var:*}/morefixed/{otherVar}", "/fixed/middle/stuff/morefixed/end", map[string]string{"var": "middle/stuff", "otherVar": "end"}}, + {"singleParam", "/fixed/{var}", "/fixed/remainder", map[string]string{"var": "remainder"}}, + {"slash", "/", "/", map[string]string{}}, + {"NoVars", "/fixed", "/fixed", map[string]string{}}, + {"TwoVars", "/from/{source}/to/{destination}", "/from/LHR/to/AMS", map[string]string{"source": "LHR", "destination": "AMS"}}, + {"VarOnFront", "/{what}/from/{source}", "/who/from/SOS", map[string]string{"what": "who", "source": "SOS"}}, +} + +func TestExtractParams(t *testing.T) { + for _, testCase := range extractParams { + t.Run(testCase.name, func(t *testing.T) { + ws1 := new(WebService).Path("/") + ws1.Route(ws1.GET(testCase.routePath).To(dummy)) + router := RouterJSR311{} + req, _ := http.NewRequest(http.MethodGet, testCase.urlPath, nil) + params := router.ExtractParameters(&ws1.Routes()[0], ws1, req.URL.Path) + if len(params) != len(testCase.expectedParams) { + t.Fatalf("Wrong length of params on selected route, expected: %v, got: %v", testCase.expectedParams, params) + } + for expectedParamKey, expectedParamValue := range testCase.expectedParams { + if expectedParamValue != params[expectedParamKey] { + t.Errorf("Wrong parameter for key '%v', expected: %v, got: %v", expectedParamKey, expectedParamValue, params[expectedParamKey]) + } + } + }) + } +} + +func TestSelectRouteInvalidMethod(t *testing.T) { + ws1 := new(WebService).Path("/") + ws1.Route(ws1.GET("/simple").To(dummy)) + router := RouterJSR311{} + req, _ := http.NewRequest(http.MethodPost, "/simple", nil) + _, _, err := router.SelectRoute([]*WebService{ws1}, req) + if err == nil { + t.Fatal("Expected an error as the wrong method is used but was nil") + } +} + +func TestParameterInWebService(t *testing.T) { + for _, testCase := range extractParams { + t.Run(testCase.name, func(t *testing.T) { + ws1 := new(WebService).Path("/{wsParam}") + ws1.Route(ws1.GET(testCase.routePath).To(dummy)) + router := RouterJSR311{} + req, _ := http.NewRequest(http.MethodGet, "/wsValue"+testCase.urlPath, nil) + params := router.ExtractParameters(&ws1.Routes()[0], ws1, req.URL.Path) + expectedParams := map[string]string{"wsParam": "wsValue"} + for key, value := range testCase.expectedParams { + expectedParams[key] = value + } + if len(params) != len(expectedParams) { + t.Fatalf("Wrong length of params on selected route, expected: %v, got: %v", testCase.expectedParams, params) + } + for expectedParamKey, expectedParamValue := range testCase.expectedParams { + if expectedParamValue != params[expectedParamKey] { + t.Errorf("Wrong parameter for key '%v', expected: %v, got: %v", expectedParamKey, expectedParamValue, params[expectedParamKey]) + } + } + }) + } +} + +func dummy(req *Request, resp *Response) { io.WriteString(resp.ResponseWriter, "dummy") } diff --git a/vendor/github.com/emicklei/go-restful/log/log.go b/vendor/github.com/emicklei/go-restful/log/log.go new file mode 100644 index 00000000000..6cd44c7a5d7 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/log/log.go @@ -0,0 +1,34 @@ +package log + +import ( + stdlog "log" + "os" +) + +// StdLogger corresponds to a minimal subset of the interface satisfied by stdlib log.Logger +type StdLogger interface { + Print(v ...interface{}) + Printf(format string, v ...interface{}) +} + +var Logger StdLogger + +func init() { + // default Logger + SetLogger(stdlog.New(os.Stderr, "[restful] ", stdlog.LstdFlags|stdlog.Lshortfile)) +} + +// SetLogger sets the logger for this package +func SetLogger(customLogger StdLogger) { + Logger = customLogger +} + +// Print delegates to the Logger +func Print(v ...interface{}) { + Logger.Print(v...) +} + +// Printf delegates to the Logger +func Printf(format string, v ...interface{}) { + Logger.Printf(format, v...) +} diff --git a/vendor/github.com/emicklei/go-restful/logger.go b/vendor/github.com/emicklei/go-restful/logger.go new file mode 100644 index 00000000000..6595df00296 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/logger.go @@ -0,0 +1,32 @@ +package restful + +// Copyright 2014 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. +import ( + "github.com/emicklei/go-restful/log" +) + +var trace bool = false +var traceLogger log.StdLogger + +func init() { + traceLogger = log.Logger // use the package logger by default +} + +// TraceLogger enables detailed logging of Http request matching and filter invocation. Default no logger is set. +// You may call EnableTracing() directly to enable trace logging to the package-wide logger. +func TraceLogger(logger log.StdLogger) { + traceLogger = logger + EnableTracing(logger != nil) +} + +// SetLogger exposes the setter for the global logger on the top-level package +func SetLogger(customLogger log.StdLogger) { + log.SetLogger(customLogger) +} + +// EnableTracing can be used to Trace logging on and off. +func EnableTracing(enabled bool) { + trace = enabled +} diff --git a/vendor/github.com/emicklei/go-restful/mime.go b/vendor/github.com/emicklei/go-restful/mime.go new file mode 100644 index 00000000000..d7ea2b61579 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/mime.go @@ -0,0 +1,45 @@ +package restful + +import ( + "strconv" + "strings" +) + +type mime struct { + media string + quality float64 +} + +// insertMime adds a mime to a list and keeps it sorted by quality. +func insertMime(l []mime, e mime) []mime { + for i, each := range l { + // if current mime has lower quality then insert before + if e.quality > each.quality { + left := append([]mime{}, l[0:i]...) + return append(append(left, e), l[i:]...) + } + } + return append(l, e) +} + +// sortedMimes returns a list of mime sorted (desc) by its specified quality. +func sortedMimes(accept string) (sorted []mime) { + for _, each := range strings.Split(accept, ",") { + typeAndQuality := strings.Split(strings.Trim(each, " "), ";") + if len(typeAndQuality) == 1 { + sorted = insertMime(sorted, mime{typeAndQuality[0], 1.0}) + } else { + // take factor + parts := strings.Split(typeAndQuality[1], "=") + if len(parts) == 2 { + f, err := strconv.ParseFloat(parts[1], 64) + if err != nil { + traceLogger.Printf("unable to parse quality in %s, %v", each, err) + } else { + sorted = insertMime(sorted, mime{typeAndQuality[0], f}) + } + } + } + } + return +} diff --git a/vendor/github.com/emicklei/go-restful/mime_test.go b/vendor/github.com/emicklei/go-restful/mime_test.go new file mode 100644 index 00000000000..a910bb1005f --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/mime_test.go @@ -0,0 +1,17 @@ +package restful + +import ( + "fmt" + "testing" +) + +// go test -v -test.run TestSortMimes ...restful +func TestSortMimes(t *testing.T) { + accept := "text/html; q=0.8, text/plain, image/gif, */*; q=0.01, image/jpeg" + result := sortedMimes(accept) + got := fmt.Sprintf("%v", result) + want := "[{text/plain 1} {image/gif 1} {image/jpeg 1} {text/html 0.8} {*/* 0.01}]" + if got != want { + t.Errorf("bad sort order of mime types:%s", got) + } +} diff --git a/vendor/github.com/emicklei/go-restful/options_filter.go b/vendor/github.com/emicklei/go-restful/options_filter.go new file mode 100644 index 00000000000..5c1b34251c1 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/options_filter.go @@ -0,0 +1,34 @@ +package restful + +import "strings" + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +// OPTIONSFilter is a filter function that inspects the Http Request for the OPTIONS method +// and provides the response with a set of allowed methods for the request URL Path. +// As for any filter, you can also install it for a particular WebService within a Container. +// Note: this filter is not needed when using CrossOriginResourceSharing (for CORS). +func (c *Container) OPTIONSFilter(req *Request, resp *Response, chain *FilterChain) { + if "OPTIONS" != req.Request.Method { + chain.ProcessFilter(req, resp) + return + } + + archs := req.Request.Header.Get(HEADER_AccessControlRequestHeaders) + methods := strings.Join(c.computeAllowedMethods(req), ",") + origin := req.Request.Header.Get(HEADER_Origin) + + resp.AddHeader(HEADER_Allow, methods) + resp.AddHeader(HEADER_AccessControlAllowOrigin, origin) + resp.AddHeader(HEADER_AccessControlAllowHeaders, archs) + resp.AddHeader(HEADER_AccessControlAllowMethods, methods) +} + +// OPTIONSFilter is a filter function that inspects the Http Request for the OPTIONS method +// and provides the response with a set of allowed methods for the request URL Path. +// Note: this filter is not needed when using CrossOriginResourceSharing (for CORS). +func OPTIONSFilter() FilterFunction { + return DefaultContainer.OPTIONSFilter +} diff --git a/vendor/github.com/emicklei/go-restful/options_filter_test.go b/vendor/github.com/emicklei/go-restful/options_filter_test.go new file mode 100644 index 00000000000..f0fceb834e6 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/options_filter_test.go @@ -0,0 +1,34 @@ +package restful + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +// go test -v -test.run TestOptionsFilter ...restful +func TestOptionsFilter(t *testing.T) { + tearDown() + ws := new(WebService) + ws.Route(ws.GET("/candy/{kind}").To(dummy)) + ws.Route(ws.DELETE("/candy/{kind}").To(dummy)) + ws.Route(ws.POST("/candies").To(dummy)) + Add(ws) + Filter(OPTIONSFilter()) + + httpRequest, _ := http.NewRequest("OPTIONS", "http://here.io/candy/gum", nil) + httpWriter := httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + actual := httpWriter.Header().Get(HEADER_Allow) + if "GET,DELETE" != actual { + t.Fatal("expected: GET,DELETE but got:" + actual) + } + + httpRequest, _ = http.NewRequest("OPTIONS", "http://here.io/candies", nil) + httpWriter = httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + actual = httpWriter.Header().Get(HEADER_Allow) + if "POST" != actual { + t.Fatal("expected: POST but got:" + actual) + } +} diff --git a/vendor/github.com/emicklei/go-restful/parameter.go b/vendor/github.com/emicklei/go-restful/parameter.go new file mode 100644 index 00000000000..e8793304b15 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/parameter.go @@ -0,0 +1,143 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +const ( + // PathParameterKind = indicator of Request parameter type "path" + PathParameterKind = iota + + // QueryParameterKind = indicator of Request parameter type "query" + QueryParameterKind + + // BodyParameterKind = indicator of Request parameter type "body" + BodyParameterKind + + // HeaderParameterKind = indicator of Request parameter type "header" + HeaderParameterKind + + // FormParameterKind = indicator of Request parameter type "form" + FormParameterKind + + // CollectionFormatCSV comma separated values `foo,bar` + CollectionFormatCSV = CollectionFormat("csv") + + // CollectionFormatSSV space separated values `foo bar` + CollectionFormatSSV = CollectionFormat("ssv") + + // CollectionFormatTSV tab separated values `foo\tbar` + CollectionFormatTSV = CollectionFormat("tsv") + + // CollectionFormatPipes pipe separated values `foo|bar` + CollectionFormatPipes = CollectionFormat("pipes") + + // CollectionFormatMulti corresponds to multiple parameter instances instead of multiple values for a single + // instance `foo=bar&foo=baz`. This is valid only for QueryParameters and FormParameters + CollectionFormatMulti = CollectionFormat("multi") +) + +type CollectionFormat string + +func (cf CollectionFormat) String() string { + return string(cf) +} + +// Parameter is for documententing the parameter used in a Http Request +// ParameterData kinds are Path,Query and Body +type Parameter struct { + data *ParameterData +} + +// ParameterData represents the state of a Parameter. +// It is made public to make it accessible to e.g. the Swagger package. +type ParameterData struct { + Name, Description, DataType, DataFormat string + Kind int + Required bool + AllowableValues map[string]string + AllowMultiple bool + DefaultValue string + CollectionFormat string +} + +// Data returns the state of the Parameter +func (p *Parameter) Data() ParameterData { + return *p.data +} + +// Kind returns the parameter type indicator (see const for valid values) +func (p *Parameter) Kind() int { + return p.data.Kind +} + +func (p *Parameter) bePath() *Parameter { + p.data.Kind = PathParameterKind + return p +} +func (p *Parameter) beQuery() *Parameter { + p.data.Kind = QueryParameterKind + return p +} +func (p *Parameter) beBody() *Parameter { + p.data.Kind = BodyParameterKind + return p +} + +func (p *Parameter) beHeader() *Parameter { + p.data.Kind = HeaderParameterKind + return p +} + +func (p *Parameter) beForm() *Parameter { + p.data.Kind = FormParameterKind + return p +} + +// Required sets the required field and returns the receiver +func (p *Parameter) Required(required bool) *Parameter { + p.data.Required = required + return p +} + +// AllowMultiple sets the allowMultiple field and returns the receiver +func (p *Parameter) AllowMultiple(multiple bool) *Parameter { + p.data.AllowMultiple = multiple + return p +} + +// AllowableValues sets the allowableValues field and returns the receiver +func (p *Parameter) AllowableValues(values map[string]string) *Parameter { + p.data.AllowableValues = values + return p +} + +// DataType sets the dataType field and returns the receiver +func (p *Parameter) DataType(typeName string) *Parameter { + p.data.DataType = typeName + return p +} + +// DataFormat sets the dataFormat field for Swagger UI +func (p *Parameter) DataFormat(formatName string) *Parameter { + p.data.DataFormat = formatName + return p +} + +// DefaultValue sets the default value field and returns the receiver +func (p *Parameter) DefaultValue(stringRepresentation string) *Parameter { + p.data.DefaultValue = stringRepresentation + return p +} + +// Description sets the description value field and returns the receiver +func (p *Parameter) Description(doc string) *Parameter { + p.data.Description = doc + return p +} + +// CollectionFormat sets the collection format for an array type +func (p *Parameter) CollectionFormat(format CollectionFormat) *Parameter { + p.data.CollectionFormat = format.String() + return p +} diff --git a/vendor/github.com/emicklei/go-restful/path_expression.go b/vendor/github.com/emicklei/go-restful/path_expression.go new file mode 100644 index 00000000000..95a9a254500 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/path_expression.go @@ -0,0 +1,74 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "bytes" + "fmt" + "regexp" + "strings" +) + +// PathExpression holds a compiled path expression (RegExp) needed to match against +// Http request paths and to extract path parameter values. +type pathExpression struct { + LiteralCount int // the number of literal characters (means those not resulting from template variable substitution) + VarNames []string // the names of parameters (enclosed by {}) in the path + VarCount int // the number of named parameters (enclosed by {}) in the path + Matcher *regexp.Regexp + Source string // Path as defined by the RouteBuilder + tokens []string +} + +// NewPathExpression creates a PathExpression from the input URL path. +// Returns an error if the path is invalid. +func newPathExpression(path string) (*pathExpression, error) { + expression, literalCount, varNames, varCount, tokens := templateToRegularExpression(path) + compiled, err := regexp.Compile(expression) + if err != nil { + return nil, err + } + return &pathExpression{literalCount, varNames, varCount, compiled, expression, tokens}, nil +} + +// http://jsr311.java.net/nonav/releases/1.1/spec/spec3.html#x3-370003.7.3 +func templateToRegularExpression(template string) (expression string, literalCount int, varNames []string, varCount int, tokens []string) { + var buffer bytes.Buffer + buffer.WriteString("^") + //tokens = strings.Split(template, "/") + tokens = tokenizePath(template) + for _, each := range tokens { + if each == "" { + continue + } + buffer.WriteString("/") + if strings.HasPrefix(each, "{") { + // check for regular expression in variable + colon := strings.Index(each, ":") + var varName string + if colon != -1 { + // extract expression + varName = strings.TrimSpace(each[1:colon]) + paramExpr := strings.TrimSpace(each[colon+1 : len(each)-1]) + if paramExpr == "*" { // special case + buffer.WriteString("(.*)") + } else { + buffer.WriteString(fmt.Sprintf("(%s)", paramExpr)) // between colon and closing moustache + } + } else { + // plain var + varName = strings.TrimSpace(each[1 : len(each)-1]) + buffer.WriteString("([^/]+?)") + } + varNames = append(varNames, varName) + varCount += 1 + } else { + literalCount += len(each) + encoded := each // TODO URI encode + buffer.WriteString(regexp.QuoteMeta(encoded)) + } + } + return strings.TrimRight(buffer.String(), "/") + "(/.*)?$", literalCount, varNames, varCount, tokens +} diff --git a/vendor/github.com/emicklei/go-restful/path_expression_test.go b/vendor/github.com/emicklei/go-restful/path_expression_test.go new file mode 100644 index 00000000000..073f1989eaa --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/path_expression_test.go @@ -0,0 +1,45 @@ +package restful + +import ( + "reflect" + "testing" +) + +var tempregexs = []struct { + template, regex string + names []string + literalCount, varCount int +}{ + {"", "^(/.*)?$", nil, 0, 0}, + {"/a/{b}/c/", "^/a/([^/]+?)/c(/.*)?$", []string{"b"}, 2, 1}, + {"/{a}/{b}/{c-d-e}/", "^/([^/]+?)/([^/]+?)/([^/]+?)(/.*)?$", []string{"a", "b", "c-d-e"}, 0, 3}, + {"/{p}/abcde", "^/([^/]+?)/abcde(/.*)?$", []string{"p"}, 5, 1}, + {"/a/{b:*}", "^/a/(.*)(/.*)?$", []string{"b"}, 1, 1}, + {"/a/{b:[a-z]+}", "^/a/([a-z]+)(/.*)?$", []string{"b"}, 1, 1}, +} + +func TestTemplateToRegularExpression(t *testing.T) { + ok := true + for i, fixture := range tempregexs { + actual, lCount, varNames, vCount, _ := templateToRegularExpression(fixture.template) + if actual != fixture.regex { + t.Logf("regex mismatch, expected:%v , actual:%v, line:%v\n", fixture.regex, actual, i) // 11 = where the data starts + ok = false + } + if lCount != fixture.literalCount { + t.Logf("literal count mismatch, expected:%v , actual:%v, line:%v\n", fixture.literalCount, lCount, i) + ok = false + } + if vCount != fixture.varCount { + t.Logf("variable count mismatch, expected:%v , actual:%v, line:%v\n", fixture.varCount, vCount, i) + ok = false + } + if !reflect.DeepEqual(fixture.names, varNames) { + t.Logf("variable name mismatch, expected:%v , actual:%v, line:%v\n", fixture.names, varNames, i) + ok = false + } + } + if !ok { + t.Fatal("one or more expression did not match") + } +} diff --git a/vendor/github.com/emicklei/go-restful/path_processor.go b/vendor/github.com/emicklei/go-restful/path_processor.go new file mode 100644 index 00000000000..357c723a7ae --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/path_processor.go @@ -0,0 +1,63 @@ +package restful + +import ( + "bytes" + "strings" +) + +// Copyright 2018 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +// PathProcessor is extra behaviour that a Router can provide to extract path parameters from the path. +// If a Router does not implement this interface then the default behaviour will be used. +type PathProcessor interface { + // ExtractParameters gets the path parameters defined in the route and webService from the urlPath + ExtractParameters(route *Route, webService *WebService, urlPath string) map[string]string +} + +type defaultPathProcessor struct{} + +// Extract the parameters from the request url path +func (d defaultPathProcessor) ExtractParameters(r *Route, _ *WebService, urlPath string) map[string]string { + urlParts := tokenizePath(urlPath) + pathParameters := map[string]string{} + for i, key := range r.pathParts { + var value string + if i >= len(urlParts) { + value = "" + } else { + value = urlParts[i] + } + if strings.HasPrefix(key, "{") { // path-parameter + if colon := strings.Index(key, ":"); colon != -1 { + // extract by regex + regPart := key[colon+1 : len(key)-1] + keyPart := key[1:colon] + if regPart == "*" { + pathParameters[keyPart] = untokenizePath(i, urlParts) + break + } else { + pathParameters[keyPart] = value + } + } else { + // without enclosing {} + pathParameters[key[1:len(key)-1]] = value + } + } + } + return pathParameters +} + +// Untokenize back into an URL path using the slash separator +func untokenizePath(offset int, parts []string) string { + var buffer bytes.Buffer + for p := offset; p < len(parts); p++ { + buffer.WriteString(parts[p]) + // do not end + if p < len(parts)-1 { + buffer.WriteString("/") + } + } + return buffer.String() +} diff --git a/vendor/github.com/emicklei/go-restful/path_processor_test.go b/vendor/github.com/emicklei/go-restful/path_processor_test.go new file mode 100644 index 00000000000..fd1b07dd78e --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/path_processor_test.go @@ -0,0 +1,55 @@ +package restful + +import "testing" + +func TestMatchesPath_OneParam(t *testing.T) { + params := doExtractParams("/from/{source}", 2, "/from/here", t) + if params["source"] != "here" { + t.Errorf("parameter mismatch here") + } +} + +func TestMatchesPath_Slash(t *testing.T) { + params := doExtractParams("/", 0, "/", t) + if len(params) != 0 { + t.Errorf("expected empty parameters") + } +} + +func TestMatchesPath_SlashNonVar(t *testing.T) { + params := doExtractParams("/any", 1, "/any", t) + if len(params) != 0 { + t.Errorf("expected empty parameters") + } +} + +func TestMatchesPath_TwoVars(t *testing.T) { + params := doExtractParams("/from/{source}/to/{destination}", 4, "/from/AMS/to/NY", t) + if params["source"] != "AMS" { + t.Errorf("parameter mismatch AMS") + } +} + +func TestMatchesPath_VarOnFront(t *testing.T) { + params := doExtractParams("{what}/from/{source}/", 3, "who/from/SOS/", t) + if params["source"] != "SOS" { + t.Errorf("parameter mismatch SOS") + } +} + +func TestExtractParameters_EmptyValue(t *testing.T) { + params := doExtractParams("/fixed/{var}", 2, "/fixed/", t) + if params["var"] != "" { + t.Errorf("parameter mismatch var") + } +} + +func doExtractParams(routePath string, size int, urlPath string, t *testing.T) map[string]string { + r := Route{Path: routePath} + r.postBuild() + if len(r.pathParts) != size { + t.Fatalf("len not %v %v, but %v", size, r.pathParts, len(r.pathParts)) + } + pathProcessor := defaultPathProcessor{} + return pathProcessor.ExtractParameters(&r, nil, urlPath) +} diff --git a/vendor/github.com/emicklei/go-restful/request.go b/vendor/github.com/emicklei/go-restful/request.go new file mode 100644 index 00000000000..a20730febf3 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/request.go @@ -0,0 +1,118 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "compress/zlib" + "net/http" +) + +var defaultRequestContentType string + +// Request is a wrapper for a http Request that provides convenience methods +type Request struct { + Request *http.Request + pathParameters map[string]string + attributes map[string]interface{} // for storing request-scoped values + selectedRoutePath string // root path + route path that matched the request, e.g. /meetings/{id}/attendees +} + +func NewRequest(httpRequest *http.Request) *Request { + return &Request{ + Request: httpRequest, + pathParameters: map[string]string{}, + attributes: map[string]interface{}{}, + } // empty parameters, attributes +} + +// If ContentType is missing or */* is given then fall back to this type, otherwise +// a "Unable to unmarshal content of type:" response is returned. +// Valid values are restful.MIME_JSON and restful.MIME_XML +// Example: +// restful.DefaultRequestContentType(restful.MIME_JSON) +func DefaultRequestContentType(mime string) { + defaultRequestContentType = mime +} + +// PathParameter accesses the Path parameter value by its name +func (r *Request) PathParameter(name string) string { + return r.pathParameters[name] +} + +// PathParameters accesses the Path parameter values +func (r *Request) PathParameters() map[string]string { + return r.pathParameters +} + +// QueryParameter returns the (first) Query parameter value by its name +func (r *Request) QueryParameter(name string) string { + return r.Request.FormValue(name) +} + +// QueryParameters returns the all the query parameters values by name +func (r *Request) QueryParameters(name string) []string { + return r.Request.URL.Query()[name] +} + +// BodyParameter parses the body of the request (once for typically a POST or a PUT) and returns the value of the given name or an error. +func (r *Request) BodyParameter(name string) (string, error) { + err := r.Request.ParseForm() + if err != nil { + return "", err + } + return r.Request.PostFormValue(name), nil +} + +// HeaderParameter returns the HTTP Header value of a Header name or empty if missing +func (r *Request) HeaderParameter(name string) string { + return r.Request.Header.Get(name) +} + +// ReadEntity checks the Accept header and reads the content into the entityPointer. +func (r *Request) ReadEntity(entityPointer interface{}) (err error) { + contentType := r.Request.Header.Get(HEADER_ContentType) + contentEncoding := r.Request.Header.Get(HEADER_ContentEncoding) + + // check if the request body needs decompression + if ENCODING_GZIP == contentEncoding { + gzipReader := currentCompressorProvider.AcquireGzipReader() + defer currentCompressorProvider.ReleaseGzipReader(gzipReader) + gzipReader.Reset(r.Request.Body) + r.Request.Body = gzipReader + } else if ENCODING_DEFLATE == contentEncoding { + zlibReader, err := zlib.NewReader(r.Request.Body) + if err != nil { + return err + } + r.Request.Body = zlibReader + } + + // lookup the EntityReader, use defaultRequestContentType if needed and provided + entityReader, ok := entityAccessRegistry.accessorAt(contentType) + if !ok { + if len(defaultRequestContentType) != 0 { + entityReader, ok = entityAccessRegistry.accessorAt(defaultRequestContentType) + } + if !ok { + return NewError(http.StatusBadRequest, "Unable to unmarshal content of type:"+contentType) + } + } + return entityReader.Read(r, entityPointer) +} + +// SetAttribute adds or replaces the attribute with the given value. +func (r *Request) SetAttribute(name string, value interface{}) { + r.attributes[name] = value +} + +// Attribute returns the value associated to the given name. Returns nil if absent. +func (r Request) Attribute(name string) interface{} { + return r.attributes[name] +} + +// SelectedRoutePath root path + route path that matched the request, e.g. /meetings/{id}/attendees +func (r Request) SelectedRoutePath() string { + return r.selectedRoutePath +} diff --git a/vendor/github.com/emicklei/go-restful/request_test.go b/vendor/github.com/emicklei/go-restful/request_test.go new file mode 100644 index 00000000000..3ab7e27eb51 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/request_test.go @@ -0,0 +1,155 @@ +package restful + +import ( + "encoding/json" + "net/http" + "net/url" + "strconv" + "strings" + "testing" +) + +func TestQueryParameter(t *testing.T) { + hreq := http.Request{Method: "GET"} + hreq.URL, _ = url.Parse("http://www.google.com/search?q=foo&q=bar") + rreq := Request{Request: &hreq} + if rreq.QueryParameter("q") != "foo" { + t.Errorf("q!=foo %#v", rreq) + } +} + +func TestQueryParameters(t *testing.T) { + hreq := http.Request{Method: "GET"} + hreq.URL, _ = url.Parse("http://www.google.com/search?q=foo&q=bar") + rreq := Request{Request: &hreq} + parameters := rreq.QueryParameters("q") + if len(parameters) != 2 { + t.Fatalf("len(q)!=2 %#v", rreq) + } else { + if parameters[0] != "foo" || parameters[1] != "bar" { + t.Fatalf("invalid content: required [\"foo\" \"bar\", got: %#v", parameters) + } + } +} + +type Anything map[string]interface{} + +type Number struct { + ValueFloat float64 + ValueInt int64 +} + +type Sample struct { + Value string +} + +func TestReadEntityJson(t *testing.T) { + bodyReader := strings.NewReader(`{"Value" : "42"}`) + httpRequest, _ := http.NewRequest("GET", "/test", bodyReader) + httpRequest.Header.Set("Content-Type", "application/json") + request := &Request{Request: httpRequest} + sam := new(Sample) + request.ReadEntity(sam) + if sam.Value != "42" { + t.Fatal("read failed") + } +} + +func TestReadEntityJsonCharset(t *testing.T) { + bodyReader := strings.NewReader(`{"Value" : "42"}`) + httpRequest, _ := http.NewRequest("GET", "/test", bodyReader) + httpRequest.Header.Set("Content-Type", "application/json; charset=UTF-8") + request := NewRequest(httpRequest) + sam := new(Sample) + request.ReadEntity(sam) + if sam.Value != "42" { + t.Fatal("read failed") + } +} + +func TestReadEntityJsonNumber(t *testing.T) { + bodyReader := strings.NewReader(`{"Value" : 4899710515899924123}`) + httpRequest, _ := http.NewRequest("GET", "/test", bodyReader) + httpRequest.Header.Set("Content-Type", "application/json") + request := &Request{Request: httpRequest} + any := make(Anything) + request.ReadEntity(&any) + number, ok := any["Value"].(json.Number) + if !ok { + t.Fatal("read failed") + } + vint, err := number.Int64() + if err != nil { + t.Fatal("convert failed") + } + if vint != 4899710515899924123 { + t.Fatal("read failed") + } + vfloat, err := number.Float64() + if err != nil { + t.Fatal("convert failed") + } + // match the default behaviour + vstring := strconv.FormatFloat(vfloat, 'e', 15, 64) + if vstring != "4.899710515899924e+18" { + t.Fatal("convert float64 failed") + } +} + +func TestReadEntityJsonLong(t *testing.T) { + bodyReader := strings.NewReader(`{"ValueFloat" : 4899710515899924123, "ValueInt": 4899710515899924123}`) + httpRequest, _ := http.NewRequest("GET", "/test", bodyReader) + httpRequest.Header.Set("Content-Type", "application/json") + request := &Request{Request: httpRequest} + number := new(Number) + request.ReadEntity(&number) + if number.ValueInt != 4899710515899924123 { + t.Fatal("read failed") + } + // match the default behaviour + vstring := strconv.FormatFloat(number.ValueFloat, 'e', 15, 64) + if vstring != "4.899710515899924e+18" { + t.Fatal("convert float64 failed") + } +} + +func TestBodyParameter(t *testing.T) { + bodyReader := strings.NewReader(`value1=42&value2=43`) + httpRequest, _ := http.NewRequest("POST", "/test?value1=44", bodyReader) // POST and PUT body parameters take precedence over URL query string + httpRequest.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") + request := NewRequest(httpRequest) + v1, err := request.BodyParameter("value1") + if err != nil { + t.Error(err) + } + v2, err := request.BodyParameter("value2") + if err != nil { + t.Error(err) + } + if v1 != "42" || v2 != "43" { + t.Fatal("read failed") + } +} + +func TestReadEntityUnkown(t *testing.T) { + bodyReader := strings.NewReader("?") + httpRequest, _ := http.NewRequest("GET", "/test", bodyReader) + httpRequest.Header.Set("Content-Type", "application/rubbish") + request := NewRequest(httpRequest) + sam := new(Sample) + err := request.ReadEntity(sam) + if err == nil { + t.Fatal("read should be in error") + } +} + +func TestSetAttribute(t *testing.T) { + bodyReader := strings.NewReader("?") + httpRequest, _ := http.NewRequest("GET", "/test", bodyReader) + request := NewRequest(httpRequest) + request.SetAttribute("go", "there") + there := request.Attribute("go") + if there != "there" { + t.Fatalf("missing request attribute:%v", there) + } +} diff --git a/vendor/github.com/emicklei/go-restful/response.go b/vendor/github.com/emicklei/go-restful/response.go new file mode 100644 index 00000000000..4d987d130b2 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/response.go @@ -0,0 +1,250 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "bufio" + "errors" + "net" + "net/http" +) + +// DefaultResponseMimeType is DEPRECATED, use DefaultResponseContentType(mime) +var DefaultResponseMimeType string + +//PrettyPrintResponses controls the indentation feature of XML and JSON serialization +var PrettyPrintResponses = true + +// Response is a wrapper on the actual http ResponseWriter +// It provides several convenience methods to prepare and write response content. +type Response struct { + http.ResponseWriter + requestAccept string // mime-type what the Http Request says it wants to receive + routeProduces []string // mime-types what the Route says it can produce + statusCode int // HTTP status code that has been written explicitly (if zero then net/http has written 200) + contentLength int // number of bytes written for the response body + prettyPrint bool // controls the indentation feature of XML and JSON serialization. It is initialized using var PrettyPrintResponses. + err error // err property is kept when WriteError is called + hijacker http.Hijacker // if underlying ResponseWriter supports it +} + +// NewResponse creates a new response based on a http ResponseWriter. +func NewResponse(httpWriter http.ResponseWriter) *Response { + hijacker, _ := httpWriter.(http.Hijacker) + return &Response{ResponseWriter: httpWriter, routeProduces: []string{}, statusCode: http.StatusOK, prettyPrint: PrettyPrintResponses, hijacker: hijacker} +} + +// DefaultResponseContentType set a default. +// If Accept header matching fails, fall back to this type. +// Valid values are restful.MIME_JSON and restful.MIME_XML +// Example: +// restful.DefaultResponseContentType(restful.MIME_JSON) +func DefaultResponseContentType(mime string) { + DefaultResponseMimeType = mime +} + +// InternalServerError writes the StatusInternalServerError header. +// DEPRECATED, use WriteErrorString(http.StatusInternalServerError,reason) +func (r Response) InternalServerError() Response { + r.WriteHeader(http.StatusInternalServerError) + return r +} + +// Hijack implements the http.Hijacker interface. This expands +// the Response to fulfill http.Hijacker if the underlying +// http.ResponseWriter supports it. +func (r *Response) Hijack() (net.Conn, *bufio.ReadWriter, error) { + if r.hijacker == nil { + return nil, nil, errors.New("http.Hijacker not implemented by underlying http.ResponseWriter") + } + return r.hijacker.Hijack() +} + +// PrettyPrint changes whether this response must produce pretty (line-by-line, indented) JSON or XML output. +func (r *Response) PrettyPrint(bePretty bool) { + r.prettyPrint = bePretty +} + +// AddHeader is a shortcut for .Header().Add(header,value) +func (r Response) AddHeader(header string, value string) Response { + r.Header().Add(header, value) + return r +} + +// SetRequestAccepts tells the response what Mime-type(s) the HTTP request said it wants to accept. Exposed for testing. +func (r *Response) SetRequestAccepts(mime string) { + r.requestAccept = mime +} + +// EntityWriter returns the registered EntityWriter that the entity (requested resource) +// can write according to what the request wants (Accept) and what the Route can produce or what the restful defaults say. +// If called before WriteEntity and WriteHeader then a false return value can be used to write a 406: Not Acceptable. +func (r *Response) EntityWriter() (EntityReaderWriter, bool) { + sorted := sortedMimes(r.requestAccept) + for _, eachAccept := range sorted { + for _, eachProduce := range r.routeProduces { + if eachProduce == eachAccept.media { + if w, ok := entityAccessRegistry.accessorAt(eachAccept.media); ok { + return w, true + } + } + } + if eachAccept.media == "*/*" { + for _, each := range r.routeProduces { + if w, ok := entityAccessRegistry.accessorAt(each); ok { + return w, true + } + } + } + } + // if requestAccept is empty + writer, ok := entityAccessRegistry.accessorAt(r.requestAccept) + if !ok { + // if not registered then fallback to the defaults (if set) + if DefaultResponseMimeType == MIME_JSON { + return entityAccessRegistry.accessorAt(MIME_JSON) + } + if DefaultResponseMimeType == MIME_XML { + return entityAccessRegistry.accessorAt(MIME_XML) + } + // Fallback to whatever the route says it can produce. + // https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html + for _, each := range r.routeProduces { + if w, ok := entityAccessRegistry.accessorAt(each); ok { + return w, true + } + } + if trace { + traceLogger.Printf("no registered EntityReaderWriter found for %s", r.requestAccept) + } + } + return writer, ok +} + +// WriteEntity calls WriteHeaderAndEntity with Http Status OK (200) +func (r *Response) WriteEntity(value interface{}) error { + return r.WriteHeaderAndEntity(http.StatusOK, value) +} + +// WriteHeaderAndEntity marshals the value using the representation denoted by the Accept Header and the registered EntityWriters. +// If no Accept header is specified (or */*) then respond with the Content-Type as specified by the first in the Route.Produces. +// If an Accept header is specified then respond with the Content-Type as specified by the first in the Route.Produces that is matched with the Accept header. +// If the value is nil then no response is send except for the Http status. You may want to call WriteHeader(http.StatusNotFound) instead. +// If there is no writer available that can represent the value in the requested MIME type then Http Status NotAcceptable is written. +// Current implementation ignores any q-parameters in the Accept Header. +// Returns an error if the value could not be written on the response. +func (r *Response) WriteHeaderAndEntity(status int, value interface{}) error { + writer, ok := r.EntityWriter() + if !ok { + r.WriteHeader(http.StatusNotAcceptable) + return nil + } + return writer.Write(r, status, value) +} + +// WriteAsXml is a convenience method for writing a value in xml (requires Xml tags on the value) +// It uses the standard encoding/xml package for marshalling the value ; not using a registered EntityReaderWriter. +func (r *Response) WriteAsXml(value interface{}) error { + return writeXML(r, http.StatusOK, MIME_XML, value) +} + +// WriteHeaderAndXml is a convenience method for writing a status and value in xml (requires Xml tags on the value) +// It uses the standard encoding/xml package for marshalling the value ; not using a registered EntityReaderWriter. +func (r *Response) WriteHeaderAndXml(status int, value interface{}) error { + return writeXML(r, status, MIME_XML, value) +} + +// WriteAsJson is a convenience method for writing a value in json. +// It uses the standard encoding/json package for marshalling the value ; not using a registered EntityReaderWriter. +func (r *Response) WriteAsJson(value interface{}) error { + return writeJSON(r, http.StatusOK, MIME_JSON, value) +} + +// WriteJson is a convenience method for writing a value in Json with a given Content-Type. +// It uses the standard encoding/json package for marshalling the value ; not using a registered EntityReaderWriter. +func (r *Response) WriteJson(value interface{}, contentType string) error { + return writeJSON(r, http.StatusOK, contentType, value) +} + +// WriteHeaderAndJson is a convenience method for writing the status and a value in Json with a given Content-Type. +// It uses the standard encoding/json package for marshalling the value ; not using a registered EntityReaderWriter. +func (r *Response) WriteHeaderAndJson(status int, value interface{}, contentType string) error { + return writeJSON(r, status, contentType, value) +} + +// WriteError write the http status and the error string on the response. +func (r *Response) WriteError(httpStatus int, err error) error { + r.err = err + return r.WriteErrorString(httpStatus, err.Error()) +} + +// WriteServiceError is a convenience method for a responding with a status and a ServiceError +func (r *Response) WriteServiceError(httpStatus int, err ServiceError) error { + r.err = err + return r.WriteHeaderAndEntity(httpStatus, err) +} + +// WriteErrorString is a convenience method for an error status with the actual error +func (r *Response) WriteErrorString(httpStatus int, errorReason string) error { + if r.err == nil { + // if not called from WriteError + r.err = errors.New(errorReason) + } + r.WriteHeader(httpStatus) + if _, err := r.Write([]byte(errorReason)); err != nil { + return err + } + return nil +} + +// Flush implements http.Flusher interface, which sends any buffered data to the client. +func (r *Response) Flush() { + if f, ok := r.ResponseWriter.(http.Flusher); ok { + f.Flush() + } else if trace { + traceLogger.Printf("ResponseWriter %v doesn't support Flush", r) + } +} + +// WriteHeader is overridden to remember the Status Code that has been written. +// Changes to the Header of the response have no effect after this. +func (r *Response) WriteHeader(httpStatus int) { + r.statusCode = httpStatus + r.ResponseWriter.WriteHeader(httpStatus) +} + +// StatusCode returns the code that has been written using WriteHeader. +func (r Response) StatusCode() int { + if 0 == r.statusCode { + // no status code has been written yet; assume OK + return http.StatusOK + } + return r.statusCode +} + +// Write writes the data to the connection as part of an HTTP reply. +// Write is part of http.ResponseWriter interface. +func (r *Response) Write(bytes []byte) (int, error) { + written, err := r.ResponseWriter.Write(bytes) + r.contentLength += written + return written, err +} + +// ContentLength returns the number of bytes written for the response content. +// Note that this value is only correct if all data is written through the Response using its Write* methods. +// Data written directly using the underlying http.ResponseWriter is not accounted for. +func (r Response) ContentLength() int { + return r.contentLength +} + +// CloseNotify is part of http.CloseNotifier interface +func (r Response) CloseNotify() <-chan bool { + return r.ResponseWriter.(http.CloseNotifier).CloseNotify() +} + +// Error returns the err created by WriteError +func (r Response) Error() error { + return r.err +} diff --git a/vendor/github.com/emicklei/go-restful/response_test.go b/vendor/github.com/emicklei/go-restful/response_test.go new file mode 100644 index 00000000000..832bc9ef2c6 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/response_test.go @@ -0,0 +1,213 @@ +package restful + +import ( + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestWriteHeader(t *testing.T) { + httpWriter := httptest.NewRecorder() + resp := Response{ResponseWriter: httpWriter, requestAccept: "*/*", routeProduces: []string{"*/*"}, prettyPrint: true} + resp.WriteHeader(123) + if resp.StatusCode() != 123 { + t.Errorf("Unexpected status code:%d", resp.StatusCode()) + } +} + +func TestNoWriteHeader(t *testing.T) { + httpWriter := httptest.NewRecorder() + resp := Response{ResponseWriter: httpWriter, requestAccept: "*/*", routeProduces: []string{"*/*"}, prettyPrint: true} + if resp.StatusCode() != http.StatusOK { + t.Errorf("Unexpected status code:%d", resp.StatusCode()) + } +} + +type food struct { + Kind string +} + +// go test -v -test.run TestMeasureContentLengthXml ...restful +func TestMeasureContentLengthXml(t *testing.T) { + httpWriter := httptest.NewRecorder() + resp := Response{ResponseWriter: httpWriter, requestAccept: "*/*", routeProduces: []string{"*/*"}, prettyPrint: true} + resp.WriteAsXml(food{"apple"}) + if resp.ContentLength() != 76 { + t.Errorf("Incorrect measured length:%d", resp.ContentLength()) + } +} + +// go test -v -test.run TestMeasureContentLengthJson ...restful +func TestMeasureContentLengthJson(t *testing.T) { + httpWriter := httptest.NewRecorder() + resp := Response{ResponseWriter: httpWriter, requestAccept: "*/*", routeProduces: []string{"*/*"}, prettyPrint: true} + resp.WriteAsJson(food{"apple"}) + if resp.ContentLength() != 20 { + t.Errorf("Incorrect measured length:%d", resp.ContentLength()) + } +} + +// go test -v -test.run TestMeasureContentLengthJsonNotPretty ...restful +func TestMeasureContentLengthJsonNotPretty(t *testing.T) { + httpWriter := httptest.NewRecorder() + resp := Response{ResponseWriter: httpWriter, requestAccept: "*/*", routeProduces: []string{"*/*"}} + resp.WriteAsJson(food{"apple"}) + if resp.ContentLength() != 17 { // 16+1 using the Encoder directly yields another /n + t.Errorf("Incorrect measured length:%d", resp.ContentLength()) + } +} + +// go test -v -test.run TestMeasureContentLengthWriteErrorString ...restful +func TestMeasureContentLengthWriteErrorString(t *testing.T) { + httpWriter := httptest.NewRecorder() + resp := Response{ResponseWriter: httpWriter, requestAccept: "*/*", routeProduces: []string{"*/*"}, prettyPrint: true} + resp.WriteErrorString(404, "Invalid") + if resp.ContentLength() != len("Invalid") { + t.Errorf("Incorrect measured length:%d", resp.ContentLength()) + } +} + +// go test -v -test.run TestStatusIsPassedToResponse ...restful +func TestStatusIsPassedToResponse(t *testing.T) { + for _, each := range []struct { + write, read int + }{ + {write: 204, read: 204}, + {write: 304, read: 304}, + {write: 200, read: 200}, + {write: 400, read: 400}, + } { + httpWriter := httptest.NewRecorder() + resp := Response{ResponseWriter: httpWriter, requestAccept: "*/*", routeProduces: []string{"*/*"}, prettyPrint: true} + resp.WriteHeader(each.write) + if got, want := httpWriter.Code, each.read; got != want { + t.Errorf("got %v want %v", got, want) + } + } +} + +// go test -v -test.run TestStatusCreatedAndContentTypeJson_Issue54 ...restful +func TestStatusCreatedAndContentTypeJson_Issue54(t *testing.T) { + httpWriter := httptest.NewRecorder() + resp := Response{ResponseWriter: httpWriter, requestAccept: "application/json", routeProduces: []string{"application/json"}, prettyPrint: true} + resp.WriteHeader(201) + resp.WriteAsJson(food{"Juicy"}) + if httpWriter.HeaderMap.Get("Content-Type") != "application/json" { + t.Errorf("Expected content type json but got:%s", httpWriter.HeaderMap.Get("Content-Type")) + } + if httpWriter.Code != 201 { + t.Errorf("Expected status 201 but got:%d", httpWriter.Code) + } +} + +type errorOnWriteRecorder struct { + *httptest.ResponseRecorder +} + +func (e errorOnWriteRecorder) Write(bytes []byte) (int, error) { + return 0, errors.New("fail") +} + +// go test -v -test.run TestLastWriteErrorCaught ...restful +func TestLastWriteErrorCaught(t *testing.T) { + httpWriter := errorOnWriteRecorder{httptest.NewRecorder()} + resp := Response{ResponseWriter: httpWriter, requestAccept: "application/json", routeProduces: []string{"application/json"}, prettyPrint: true} + err := resp.WriteAsJson(food{"Juicy"}) + if err.Error() != "fail" { + t.Errorf("Unexpected error message:%v", err) + } +} + +// go test -v -test.run TestAcceptStarStar_Issue83 ...restful +func TestAcceptStarStar_Issue83(t *testing.T) { + httpWriter := httptest.NewRecorder() + // Accept Produces + resp := Response{ResponseWriter: httpWriter, requestAccept: "application/bogus,*/*;q=0.8", routeProduces: []string{"application/json"}, prettyPrint: true} + resp.WriteEntity(food{"Juicy"}) + ct := httpWriter.Header().Get("Content-Type") + if "application/json" != ct { + t.Errorf("Unexpected content type:%s", ct) + } +} + +// go test -v -test.run TestAcceptSkipStarStar_Issue83 ...restful +func TestAcceptSkipStarStar_Issue83(t *testing.T) { + httpWriter := httptest.NewRecorder() + // Accept Produces + resp := Response{ResponseWriter: httpWriter, requestAccept: " application/xml ,*/* ; q=0.8", routeProduces: []string{"application/json", "application/xml"}, prettyPrint: true} + resp.WriteEntity(food{"Juicy"}) + ct := httpWriter.Header().Get("Content-Type") + if "application/xml" != ct { + t.Errorf("Unexpected content type:%s", ct) + } +} + +// go test -v -test.run TestAcceptXmlBeforeStarStar_Issue83 ...restful +func TestAcceptXmlBeforeStarStar_Issue83(t *testing.T) { + httpWriter := httptest.NewRecorder() + // Accept Produces + resp := Response{ResponseWriter: httpWriter, requestAccept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", routeProduces: []string{"application/json"}, prettyPrint: true} + resp.WriteEntity(food{"Juicy"}) + ct := httpWriter.Header().Get("Content-Type") + if "application/json" != ct { + t.Errorf("Unexpected content type:%s", ct) + } +} + +// go test -v -test.run TestWriteHeaderNoContent_Issue124 ...restful +func TestWriteHeaderNoContent_Issue124(t *testing.T) { + httpWriter := httptest.NewRecorder() + resp := Response{ResponseWriter: httpWriter, requestAccept: "text/plain", routeProduces: []string{"text/plain"}, prettyPrint: true} + resp.WriteHeader(http.StatusNoContent) + if httpWriter.Code != http.StatusNoContent { + t.Errorf("got %d want %d", httpWriter.Code, http.StatusNoContent) + } +} + +// go test -v -test.run TestStatusCreatedAndContentTypeJson_Issue163 ...restful +func TestStatusCreatedAndContentTypeJson_Issue163(t *testing.T) { + httpWriter := httptest.NewRecorder() + resp := Response{ResponseWriter: httpWriter, requestAccept: "application/json", routeProduces: []string{"application/json"}, prettyPrint: true} + resp.WriteHeader(http.StatusNotModified) + if httpWriter.Code != http.StatusNotModified { + t.Errorf("Got %d want %d", httpWriter.Code, http.StatusNotModified) + } +} + +func TestWriteHeaderAndEntity_Issue235(t *testing.T) { + httpWriter := httptest.NewRecorder() + resp := Response{ResponseWriter: httpWriter, requestAccept: "application/json", routeProduces: []string{"application/json"}, prettyPrint: true} + var pong = struct { + Foo string `json:"foo"` + }{Foo: "123"} + resp.WriteHeaderAndEntity(404, pong) + if httpWriter.Code != http.StatusNotFound { + t.Errorf("got %d want %d", httpWriter.Code, http.StatusNoContent) + } + if got, want := httpWriter.Header().Get("Content-Type"), "application/json"; got != want { + t.Errorf("got %v want %v", got, want) + } + if !strings.HasPrefix(httpWriter.Body.String(), "{") { + t.Errorf("expected pong struct in json:%s", httpWriter.Body.String()) + } +} + +func TestWriteEntityNoAcceptMatchWithProduces(t *testing.T) { + httpWriter := httptest.NewRecorder() + resp := Response{ResponseWriter: httpWriter, requestAccept: "application/bogus", routeProduces: []string{"application/json"}, prettyPrint: true} + resp.WriteEntity("done") + if httpWriter.Code != http.StatusOK { + t.Errorf("got %d want %d", httpWriter.Code, http.StatusOK) + } +} + +func TestWriteEntityNoAcceptMatchNoProduces(t *testing.T) { + httpWriter := httptest.NewRecorder() + resp := Response{ResponseWriter: httpWriter, requestAccept: "application/bogus", routeProduces: []string{}, prettyPrint: true} + resp.WriteEntity("done") + if httpWriter.Code != http.StatusNotAcceptable { + t.Errorf("got %d want %d", httpWriter.Code, http.StatusNotAcceptable) + } +} diff --git a/vendor/github.com/emicklei/go-restful/route.go b/vendor/github.com/emicklei/go-restful/route.go new file mode 100644 index 00000000000..f72bf985079 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/route.go @@ -0,0 +1,149 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "net/http" + "strings" +) + +// RouteFunction declares the signature of a function that can be bound to a Route. +type RouteFunction func(*Request, *Response) + +// RouteSelectionConditionFunction declares the signature of a function that +// can be used to add extra conditional logic when selecting whether the route +// matches the HTTP request. +type RouteSelectionConditionFunction func(httpRequest *http.Request) bool + +// Route binds a HTTP Method,Path,Consumes combination to a RouteFunction. +type Route struct { + Method string + Produces []string + Consumes []string + Path string // webservice root path + described path + Function RouteFunction + Filters []FilterFunction + If []RouteSelectionConditionFunction + + // cached values for dispatching + relativePath string + pathParts []string + pathExpr *pathExpression // cached compilation of relativePath as RegExp + + // documentation + Doc string + Notes string + Operation string + ParameterDocs []*Parameter + ResponseErrors map[int]ResponseError + ReadSample, WriteSample interface{} // structs that model an example request or response payload + + // Extra information used to store custom information about the route. + Metadata map[string]interface{} + + // marks a route as deprecated + Deprecated bool +} + +// Initialize for Route +func (r *Route) postBuild() { + r.pathParts = tokenizePath(r.Path) +} + +// Create Request and Response from their http versions +func (r *Route) wrapRequestResponse(httpWriter http.ResponseWriter, httpRequest *http.Request, pathParams map[string]string) (*Request, *Response) { + wrappedRequest := NewRequest(httpRequest) + wrappedRequest.pathParameters = pathParams + wrappedRequest.selectedRoutePath = r.Path + wrappedResponse := NewResponse(httpWriter) + wrappedResponse.requestAccept = httpRequest.Header.Get(HEADER_Accept) + wrappedResponse.routeProduces = r.Produces + return wrappedRequest, wrappedResponse +} + +// dispatchWithFilters call the function after passing through its own filters +func (r *Route) dispatchWithFilters(wrappedRequest *Request, wrappedResponse *Response) { + if len(r.Filters) > 0 { + chain := FilterChain{Filters: r.Filters, Target: r.Function} + chain.ProcessFilter(wrappedRequest, wrappedResponse) + } else { + // unfiltered + r.Function(wrappedRequest, wrappedResponse) + } +} + +// Return whether the mimeType matches to what this Route can produce. +func (r Route) matchesAccept(mimeTypesWithQuality string) bool { + parts := strings.Split(mimeTypesWithQuality, ",") + for _, each := range parts { + var withoutQuality string + if strings.Contains(each, ";") { + withoutQuality = strings.Split(each, ";")[0] + } else { + withoutQuality = each + } + // trim before compare + withoutQuality = strings.Trim(withoutQuality, " ") + if withoutQuality == "*/*" { + return true + } + for _, producibleType := range r.Produces { + if producibleType == "*/*" || producibleType == withoutQuality { + return true + } + } + } + return false +} + +// Return whether this Route can consume content with a type specified by mimeTypes (can be empty). +func (r Route) matchesContentType(mimeTypes string) bool { + + if len(r.Consumes) == 0 { + // did not specify what it can consume ; any media type (“*/*”) is assumed + return true + } + + if len(mimeTypes) == 0 { + // idempotent methods with (most-likely or guaranteed) empty content match missing Content-Type + m := r.Method + if m == "GET" || m == "HEAD" || m == "OPTIONS" || m == "DELETE" || m == "TRACE" { + return true + } + // proceed with default + mimeTypes = MIME_OCTET + } + + parts := strings.Split(mimeTypes, ",") + for _, each := range parts { + var contentType string + if strings.Contains(each, ";") { + contentType = strings.Split(each, ";")[0] + } else { + contentType = each + } + // trim before compare + contentType = strings.Trim(contentType, " ") + for _, consumeableType := range r.Consumes { + if consumeableType == "*/*" || consumeableType == contentType { + return true + } + } + } + return false +} + +// Tokenize an URL path using the slash separator ; the result does not have empty tokens +func tokenizePath(path string) []string { + if "/" == path { + return []string{} + } + return strings.Split(strings.Trim(path, "/"), "/") +} + +// for debugging +func (r Route) String() string { + return r.Method + " " + r.Path +} diff --git a/vendor/github.com/emicklei/go-restful/route_builder.go b/vendor/github.com/emicklei/go-restful/route_builder.go new file mode 100644 index 00000000000..4ebecbd8c41 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/route_builder.go @@ -0,0 +1,321 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "fmt" + "os" + "reflect" + "runtime" + "strings" + "sync/atomic" + + "github.com/emicklei/go-restful/log" +) + +// RouteBuilder is a helper to construct Routes. +type RouteBuilder struct { + rootPath string + currentPath string + produces []string + consumes []string + httpMethod string // required + function RouteFunction // required + filters []FilterFunction + conditions []RouteSelectionConditionFunction + + typeNameHandleFunc TypeNameHandleFunction // required + + // documentation + doc string + notes string + operation string + readSample, writeSample interface{} + parameters []*Parameter + errorMap map[int]ResponseError + metadata map[string]interface{} + deprecated bool +} + +// Do evaluates each argument with the RouteBuilder itself. +// This allows you to follow DRY principles without breaking the fluent programming style. +// Example: +// ws.Route(ws.DELETE("/{name}").To(t.deletePerson).Do(Returns200, Returns500)) +// +// func Returns500(b *RouteBuilder) { +// b.Returns(500, "Internal Server Error", restful.ServiceError{}) +// } +func (b *RouteBuilder) Do(oneArgBlocks ...func(*RouteBuilder)) *RouteBuilder { + for _, each := range oneArgBlocks { + each(b) + } + return b +} + +// To bind the route to a function. +// If this route is matched with the incoming Http Request then call this function with the *Request,*Response pair. Required. +func (b *RouteBuilder) To(function RouteFunction) *RouteBuilder { + b.function = function + return b +} + +// Method specifies what HTTP method to match. Required. +func (b *RouteBuilder) Method(method string) *RouteBuilder { + b.httpMethod = method + return b +} + +// Produces specifies what MIME types can be produced ; the matched one will appear in the Content-Type Http header. +func (b *RouteBuilder) Produces(mimeTypes ...string) *RouteBuilder { + b.produces = mimeTypes + return b +} + +// Consumes specifies what MIME types can be consumes ; the Accept Http header must matched any of these +func (b *RouteBuilder) Consumes(mimeTypes ...string) *RouteBuilder { + b.consumes = mimeTypes + return b +} + +// Path specifies the relative (w.r.t WebService root path) URL path to match. Default is "/". +func (b *RouteBuilder) Path(subPath string) *RouteBuilder { + b.currentPath = subPath + return b +} + +// Doc tells what this route is all about. Optional. +func (b *RouteBuilder) Doc(documentation string) *RouteBuilder { + b.doc = documentation + return b +} + +// Notes is a verbose explanation of the operation behavior. Optional. +func (b *RouteBuilder) Notes(notes string) *RouteBuilder { + b.notes = notes + return b +} + +// Reads tells what resource type will be read from the request payload. Optional. +// A parameter of type "body" is added ,required is set to true and the dataType is set to the qualified name of the sample's type. +func (b *RouteBuilder) Reads(sample interface{}, optionalDescription ...string) *RouteBuilder { + fn := b.typeNameHandleFunc + if fn == nil { + fn = reflectTypeName + } + typeAsName := fn(sample) + description := "" + if len(optionalDescription) > 0 { + description = optionalDescription[0] + } + b.readSample = sample + bodyParameter := &Parameter{&ParameterData{Name: "body", Description: description}} + bodyParameter.beBody() + bodyParameter.Required(true) + bodyParameter.DataType(typeAsName) + b.Param(bodyParameter) + return b +} + +// ParameterNamed returns a Parameter already known to the RouteBuilder. Returns nil if not. +// Use this to modify or extend information for the Parameter (through its Data()). +func (b RouteBuilder) ParameterNamed(name string) (p *Parameter) { + for _, each := range b.parameters { + if each.Data().Name == name { + return each + } + } + return p +} + +// Writes tells what resource type will be written as the response payload. Optional. +func (b *RouteBuilder) Writes(sample interface{}) *RouteBuilder { + b.writeSample = sample + return b +} + +// Param allows you to document the parameters of the Route. It adds a new Parameter (does not check for duplicates). +func (b *RouteBuilder) Param(parameter *Parameter) *RouteBuilder { + if b.parameters == nil { + b.parameters = []*Parameter{} + } + b.parameters = append(b.parameters, parameter) + return b +} + +// Operation allows you to document what the actual method/function call is of the Route. +// Unless called, the operation name is derived from the RouteFunction set using To(..). +func (b *RouteBuilder) Operation(name string) *RouteBuilder { + b.operation = name + return b +} + +// ReturnsError is deprecated, use Returns instead. +func (b *RouteBuilder) ReturnsError(code int, message string, model interface{}) *RouteBuilder { + log.Print("ReturnsError is deprecated, use Returns instead.") + return b.Returns(code, message, model) +} + +// Returns allows you to document what responses (errors or regular) can be expected. +// The model parameter is optional ; either pass a struct instance or use nil if not applicable. +func (b *RouteBuilder) Returns(code int, message string, model interface{}) *RouteBuilder { + err := ResponseError{ + Code: code, + Message: message, + Model: model, + IsDefault: false, + } + // lazy init because there is no NewRouteBuilder (yet) + if b.errorMap == nil { + b.errorMap = map[int]ResponseError{} + } + b.errorMap[code] = err + return b +} + +// DefaultReturns is a special Returns call that sets the default of the response ; the code is zero. +func (b *RouteBuilder) DefaultReturns(message string, model interface{}) *RouteBuilder { + b.Returns(0, message, model) + // Modify the ResponseError just added/updated + re := b.errorMap[0] + // errorMap is initialized + b.errorMap[0] = ResponseError{ + Code: re.Code, + Message: re.Message, + Model: re.Model, + IsDefault: true, + } + return b +} + +// Metadata adds or updates a key=value pair to the metadata map. +func (b *RouteBuilder) Metadata(key string, value interface{}) *RouteBuilder { + if b.metadata == nil { + b.metadata = map[string]interface{}{} + } + b.metadata[key] = value + return b +} + +// Deprecate sets the value of deprecated to true. Deprecated routes have a special UI treatment to warn against use +func (b *RouteBuilder) Deprecate() *RouteBuilder { + b.deprecated = true + return b +} + +// ResponseError represents a response; not necessarily an error. +type ResponseError struct { + Code int + Message string + Model interface{} + IsDefault bool +} + +func (b *RouteBuilder) servicePath(path string) *RouteBuilder { + b.rootPath = path + return b +} + +// Filter appends a FilterFunction to the end of filters for this Route to build. +func (b *RouteBuilder) Filter(filter FilterFunction) *RouteBuilder { + b.filters = append(b.filters, filter) + return b +} + +// If sets a condition function that controls matching the Route based on custom logic. +// The condition function is provided the HTTP request and should return true if the route +// should be considered. +// +// Efficiency note: the condition function is called before checking the method, produces, and +// consumes criteria, so that the correct HTTP status code can be returned. +// +// Lifecycle note: no filter functions have been called prior to calling the condition function, +// so the condition function should not depend on any context that might be set up by container +// or route filters. +func (b *RouteBuilder) If(condition RouteSelectionConditionFunction) *RouteBuilder { + b.conditions = append(b.conditions, condition) + return b +} + +// If no specific Route path then set to rootPath +// If no specific Produces then set to rootProduces +// If no specific Consumes then set to rootConsumes +func (b *RouteBuilder) copyDefaults(rootProduces, rootConsumes []string) { + if len(b.produces) == 0 { + b.produces = rootProduces + } + if len(b.consumes) == 0 { + b.consumes = rootConsumes + } +} + +// typeNameHandler sets the function that will convert types to strings in the parameter +// and model definitions. +func (b *RouteBuilder) typeNameHandler(handler TypeNameHandleFunction) *RouteBuilder { + b.typeNameHandleFunc = handler + return b +} + +// Build creates a new Route using the specification details collected by the RouteBuilder +func (b *RouteBuilder) Build() Route { + pathExpr, err := newPathExpression(b.currentPath) + if err != nil { + log.Printf("[restful] Invalid path:%s because:%v", b.currentPath, err) + os.Exit(1) + } + if b.function == nil { + log.Printf("[restful] No function specified for route:" + b.currentPath) + os.Exit(1) + } + operationName := b.operation + if len(operationName) == 0 && b.function != nil { + // extract from definition + operationName = nameOfFunction(b.function) + } + route := Route{ + Method: b.httpMethod, + Path: concatPath(b.rootPath, b.currentPath), + Produces: b.produces, + Consumes: b.consumes, + Function: b.function, + Filters: b.filters, + If: b.conditions, + relativePath: b.currentPath, + pathExpr: pathExpr, + Doc: b.doc, + Notes: b.notes, + Operation: operationName, + ParameterDocs: b.parameters, + ResponseErrors: b.errorMap, + ReadSample: b.readSample, + WriteSample: b.writeSample, + Metadata: b.metadata, + Deprecated: b.deprecated} + route.postBuild() + return route +} + +func concatPath(path1, path2 string) string { + return strings.TrimRight(path1, "/") + "/" + strings.TrimLeft(path2, "/") +} + +var anonymousFuncCount int32 + +// nameOfFunction returns the short name of the function f for documentation. +// It uses a runtime feature for debugging ; its value may change for later Go versions. +func nameOfFunction(f interface{}) string { + fun := runtime.FuncForPC(reflect.ValueOf(f).Pointer()) + tokenized := strings.Split(fun.Name(), ".") + last := tokenized[len(tokenized)-1] + last = strings.TrimSuffix(last, ")·fm") // < Go 1.5 + last = strings.TrimSuffix(last, ")-fm") // Go 1.5 + last = strings.TrimSuffix(last, "·fm") // < Go 1.5 + last = strings.TrimSuffix(last, "-fm") // Go 1.5 + if last == "func1" { // this could mean conflicts in API docs + val := atomic.AddInt32(&anonymousFuncCount, 1) + last = "func" + fmt.Sprintf("%d", val) + atomic.StoreInt32(&anonymousFuncCount, val) + } + return last +} diff --git a/vendor/github.com/emicklei/go-restful/route_builder_test.go b/vendor/github.com/emicklei/go-restful/route_builder_test.go new file mode 100644 index 00000000000..25881d5eb82 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/route_builder_test.go @@ -0,0 +1,76 @@ +package restful + +import ( + "testing" + "time" +) + +func TestRouteBuilder_PathParameter(t *testing.T) { + p := &Parameter{&ParameterData{Name: "name", Description: "desc"}} + p.AllowMultiple(true) + p.DataType("int") + p.Required(true) + values := map[string]string{"a": "b"} + p.AllowableValues(values) + p.bePath() + + b := new(RouteBuilder) + b.function = dummy + b.Param(p) + r := b.Build() + if !r.ParameterDocs[0].Data().AllowMultiple { + t.Error("AllowMultiple invalid") + } + if r.ParameterDocs[0].Data().DataType != "int" { + t.Error("dataType invalid") + } + if !r.ParameterDocs[0].Data().Required { + t.Error("required invalid") + } + if r.ParameterDocs[0].Data().Kind != PathParameterKind { + t.Error("kind invalid") + } + if r.ParameterDocs[0].Data().AllowableValues["a"] != "b" { + t.Error("allowableValues invalid") + } + if b.ParameterNamed("name") == nil { + t.Error("access to parameter failed") + } +} + +func TestRouteBuilder(t *testing.T) { + json := "application/json" + b := new(RouteBuilder) + b.To(dummy) + b.Path("/routes").Method("HEAD").Consumes(json).Produces(json).Metadata("test", "test-value").DefaultReturns("default", time.Now()) + r := b.Build() + if r.Path != "/routes" { + t.Error("path invalid") + } + if r.Produces[0] != json { + t.Error("produces invalid") + } + if r.Consumes[0] != json { + t.Error("consumes invalid") + } + if r.Operation != "dummy" { + t.Error("Operation not set") + } + if r.Metadata["test"] != "test-value" { + t.Errorf("Metadata not set") + } + if _, ok := r.ResponseErrors[0]; !ok { + t.Fatal("expected default response") + } +} + +func TestAnonymousFuncNaming(t *testing.T) { + f1 := func() {} + f2 := func() {} + if got, want := nameOfFunction(f1), "func1"; got != want { + t.Errorf("got %v want %v", got, want) + } + if got, want := nameOfFunction(f2), "func2"; got != want { + t.Errorf("got %v want %v", got, want) + } +} diff --git a/vendor/github.com/emicklei/go-restful/route_test.go b/vendor/github.com/emicklei/go-restful/route_test.go new file mode 100644 index 00000000000..a687d8a4d1b --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/route_test.go @@ -0,0 +1,76 @@ +package restful + +import ( + "testing" +) + +// accept should match produces +func TestMatchesAcceptPlainTextWhenProducePlainTextAsLast(t *testing.T) { + r := Route{Produces: []string{"application/json", "text/plain"}} + if !r.matchesAccept("text/plain") { + t.Errorf("accept should match text/plain") + } +} + +// accept should match produces +func TestMatchesAcceptStar(t *testing.T) { + r := Route{Produces: []string{"application/xml"}} + if !r.matchesAccept("*/*") { + t.Errorf("accept should match star") + } +} + +// accept should match produces +func TestMatchesAcceptIE(t *testing.T) { + r := Route{Produces: []string{"application/xml"}} + if !r.matchesAccept("text/html, application/xhtml+xml, */*") { + t.Errorf("accept should match star") + } +} + +// accept should match produces +func TestMatchesAcceptXml(t *testing.T) { + r := Route{Produces: []string{"application/xml"}} + if r.matchesAccept("application/json") { + t.Errorf("accept should not match json") + } + if !r.matchesAccept("application/xml") { + t.Errorf("accept should match xml") + } +} + +// accept should match produces +func TestMatchesAcceptAny(t *testing.T) { + r := Route{Produces: []string{"*/*"}} + if !r.matchesAccept("application/json") { + t.Errorf("accept should match json") + } + if !r.matchesAccept("application/xml") { + t.Errorf("accept should match xml") + } +} + +// content type should match consumes +func TestMatchesContentTypeXml(t *testing.T) { + r := Route{Consumes: []string{"application/xml"}} + if r.matchesContentType("application/json") { + t.Errorf("accept should not match json") + } + if !r.matchesContentType("application/xml") { + t.Errorf("accept should match xml") + } +} + +// content type should match consumes +func TestMatchesContentTypeCharsetInformation(t *testing.T) { + r := Route{Consumes: []string{"application/json"}} + if !r.matchesContentType("application/json; charset=UTF-8") { + t.Errorf("matchesContentType should ignore charset information") + } +} + +func TestTokenizePath(t *testing.T) { + if len(tokenizePath("/")) != 0 { + t.Errorf("not empty path tokens") + } +} diff --git a/vendor/github.com/emicklei/go-restful/router.go b/vendor/github.com/emicklei/go-restful/router.go new file mode 100644 index 00000000000..19078af1c06 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/router.go @@ -0,0 +1,20 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import "net/http" + +// A RouteSelector finds the best matching Route given the input HTTP Request +// RouteSelectors can optionally also implement the PathProcessor interface to also calculate the +// path parameters after the route has been selected. +type RouteSelector interface { + + // SelectRoute finds a Route given the input HTTP Request and a list of WebServices. + // It returns a selected Route and its containing WebService or an error indicating + // a problem. + SelectRoute( + webServices []*WebService, + httpRequest *http.Request) (selectedService *WebService, selected *Route, err error) +} diff --git a/vendor/github.com/emicklei/go-restful/service_error.go b/vendor/github.com/emicklei/go-restful/service_error.go new file mode 100644 index 00000000000..62d1108bbda --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/service_error.go @@ -0,0 +1,23 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import "fmt" + +// ServiceError is a transport object to pass information about a non-Http error occurred in a WebService while processing a request. +type ServiceError struct { + Code int + Message string +} + +// NewError returns a ServiceError using the code and reason +func NewError(code int, message string) ServiceError { + return ServiceError{Code: code, Message: message} +} + +// Error returns a text representation of the service error +func (s ServiceError) Error() string { + return fmt.Sprintf("[ServiceError:%v] %v", s.Code, s.Message) +} diff --git a/vendor/github.com/emicklei/go-restful/tracer_test.go b/vendor/github.com/emicklei/go-restful/tracer_test.go new file mode 100644 index 00000000000..60c1e9fc09d --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/tracer_test.go @@ -0,0 +1,18 @@ +package restful + +import "testing" + +// Use like this: +// +// TraceLogger(testLogger{t}) +type testLogger struct { + t *testing.T +} + +func (l testLogger) Print(v ...interface{}) { + l.t.Log(v...) +} + +func (l testLogger) Printf(format string, v ...interface{}) { + l.t.Logf(format, v...) +} diff --git a/vendor/github.com/emicklei/go-restful/web_service.go b/vendor/github.com/emicklei/go-restful/web_service.go new file mode 100644 index 00000000000..f7e18a5859c --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/web_service.go @@ -0,0 +1,290 @@ +package restful + +import ( + "errors" + "os" + "reflect" + "sync" + + "github.com/emicklei/go-restful/log" +) + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +// WebService holds a collection of Route values that bind a Http Method + URL Path to a function. +type WebService struct { + rootPath string + pathExpr *pathExpression // cached compilation of rootPath as RegExp + routes []Route + produces []string + consumes []string + pathParameters []*Parameter + filters []FilterFunction + documentation string + apiVersion string + + typeNameHandleFunc TypeNameHandleFunction + + dynamicRoutes bool + + // protects 'routes' if dynamic routes are enabled + routesLock sync.RWMutex +} + +func (w *WebService) SetDynamicRoutes(enable bool) { + w.dynamicRoutes = enable +} + +// TypeNameHandleFunction declares functions that can handle translating the name of a sample object +// into the restful documentation for the service. +type TypeNameHandleFunction func(sample interface{}) string + +// TypeNameHandler sets the function that will convert types to strings in the parameter +// and model definitions. If not set, the web service will invoke +// reflect.TypeOf(object).String(). +func (w *WebService) TypeNameHandler(handler TypeNameHandleFunction) *WebService { + w.typeNameHandleFunc = handler + return w +} + +// reflectTypeName is the default TypeNameHandleFunction and for a given object +// returns the name that Go identifies it with (e.g. "string" or "v1.Object") via +// the reflection API. +func reflectTypeName(sample interface{}) string { + return reflect.TypeOf(sample).String() +} + +// compilePathExpression ensures that the path is compiled into a RegEx for those routers that need it. +func (w *WebService) compilePathExpression() { + compiled, err := newPathExpression(w.rootPath) + if err != nil { + log.Printf("[restful] invalid path:%s because:%v", w.rootPath, err) + os.Exit(1) + } + w.pathExpr = compiled +} + +// ApiVersion sets the API version for documentation purposes. +func (w *WebService) ApiVersion(apiVersion string) *WebService { + w.apiVersion = apiVersion + return w +} + +// Version returns the API version for documentation purposes. +func (w *WebService) Version() string { return w.apiVersion } + +// Path specifies the root URL template path of the WebService. +// All Routes will be relative to this path. +func (w *WebService) Path(root string) *WebService { + w.rootPath = root + if len(w.rootPath) == 0 { + w.rootPath = "/" + } + w.compilePathExpression() + return w +} + +// Param adds a PathParameter to document parameters used in the root path. +func (w *WebService) Param(parameter *Parameter) *WebService { + if w.pathParameters == nil { + w.pathParameters = []*Parameter{} + } + w.pathParameters = append(w.pathParameters, parameter) + return w +} + +// PathParameter creates a new Parameter of kind Path for documentation purposes. +// It is initialized as required with string as its DataType. +func (w *WebService) PathParameter(name, description string) *Parameter { + return PathParameter(name, description) +} + +// PathParameter creates a new Parameter of kind Path for documentation purposes. +// It is initialized as required with string as its DataType. +func PathParameter(name, description string) *Parameter { + p := &Parameter{&ParameterData{Name: name, Description: description, Required: true, DataType: "string"}} + p.bePath() + return p +} + +// QueryParameter creates a new Parameter of kind Query for documentation purposes. +// It is initialized as not required with string as its DataType. +func (w *WebService) QueryParameter(name, description string) *Parameter { + return QueryParameter(name, description) +} + +// QueryParameter creates a new Parameter of kind Query for documentation purposes. +// It is initialized as not required with string as its DataType. +func QueryParameter(name, description string) *Parameter { + p := &Parameter{&ParameterData{Name: name, Description: description, Required: false, DataType: "string", CollectionFormat: CollectionFormatCSV.String()}} + p.beQuery() + return p +} + +// BodyParameter creates a new Parameter of kind Body for documentation purposes. +// It is initialized as required without a DataType. +func (w *WebService) BodyParameter(name, description string) *Parameter { + return BodyParameter(name, description) +} + +// BodyParameter creates a new Parameter of kind Body for documentation purposes. +// It is initialized as required without a DataType. +func BodyParameter(name, description string) *Parameter { + p := &Parameter{&ParameterData{Name: name, Description: description, Required: true}} + p.beBody() + return p +} + +// HeaderParameter creates a new Parameter of kind (Http) Header for documentation purposes. +// It is initialized as not required with string as its DataType. +func (w *WebService) HeaderParameter(name, description string) *Parameter { + return HeaderParameter(name, description) +} + +// HeaderParameter creates a new Parameter of kind (Http) Header for documentation purposes. +// It is initialized as not required with string as its DataType. +func HeaderParameter(name, description string) *Parameter { + p := &Parameter{&ParameterData{Name: name, Description: description, Required: false, DataType: "string"}} + p.beHeader() + return p +} + +// FormParameter creates a new Parameter of kind Form (using application/x-www-form-urlencoded) for documentation purposes. +// It is initialized as required with string as its DataType. +func (w *WebService) FormParameter(name, description string) *Parameter { + return FormParameter(name, description) +} + +// FormParameter creates a new Parameter of kind Form (using application/x-www-form-urlencoded) for documentation purposes. +// It is initialized as required with string as its DataType. +func FormParameter(name, description string) *Parameter { + p := &Parameter{&ParameterData{Name: name, Description: description, Required: false, DataType: "string"}} + p.beForm() + return p +} + +// Route creates a new Route using the RouteBuilder and add to the ordered list of Routes. +func (w *WebService) Route(builder *RouteBuilder) *WebService { + w.routesLock.Lock() + defer w.routesLock.Unlock() + builder.copyDefaults(w.produces, w.consumes) + w.routes = append(w.routes, builder.Build()) + return w +} + +// RemoveRoute removes the specified route, looks for something that matches 'path' and 'method' +func (w *WebService) RemoveRoute(path, method string) error { + if !w.dynamicRoutes { + return errors.New("dynamic routes are not enabled.") + } + w.routesLock.Lock() + defer w.routesLock.Unlock() + newRoutes := make([]Route, (len(w.routes) - 1)) + current := 0 + for ix := range w.routes { + if w.routes[ix].Method == method && w.routes[ix].Path == path { + continue + } + newRoutes[current] = w.routes[ix] + current = current + 1 + } + w.routes = newRoutes + return nil +} + +// Method creates a new RouteBuilder and initialize its http method +func (w *WebService) Method(httpMethod string) *RouteBuilder { + return new(RouteBuilder).typeNameHandler(w.typeNameHandleFunc).servicePath(w.rootPath).Method(httpMethod) +} + +// Produces specifies that this WebService can produce one or more MIME types. +// Http requests must have one of these values set for the Accept header. +func (w *WebService) Produces(contentTypes ...string) *WebService { + w.produces = contentTypes + return w +} + +// Consumes specifies that this WebService can consume one or more MIME types. +// Http requests must have one of these values set for the Content-Type header. +func (w *WebService) Consumes(accepts ...string) *WebService { + w.consumes = accepts + return w +} + +// Routes returns the Routes associated with this WebService +func (w *WebService) Routes() []Route { + if !w.dynamicRoutes { + return w.routes + } + // Make a copy of the array to prevent concurrency problems + w.routesLock.RLock() + defer w.routesLock.RUnlock() + result := make([]Route, len(w.routes)) + for ix := range w.routes { + result[ix] = w.routes[ix] + } + return result +} + +// RootPath returns the RootPath associated with this WebService. Default "/" +func (w *WebService) RootPath() string { + return w.rootPath +} + +// PathParameters return the path parameter names for (shared among its Routes) +func (w *WebService) PathParameters() []*Parameter { + return w.pathParameters +} + +// Filter adds a filter function to the chain of filters applicable to all its Routes +func (w *WebService) Filter(filter FilterFunction) *WebService { + w.filters = append(w.filters, filter) + return w +} + +// Doc is used to set the documentation of this service. +func (w *WebService) Doc(plainText string) *WebService { + w.documentation = plainText + return w +} + +// Documentation returns it. +func (w *WebService) Documentation() string { + return w.documentation +} + +/* + Convenience methods +*/ + +// HEAD is a shortcut for .Method("HEAD").Path(subPath) +func (w *WebService) HEAD(subPath string) *RouteBuilder { + return new(RouteBuilder).typeNameHandler(w.typeNameHandleFunc).servicePath(w.rootPath).Method("HEAD").Path(subPath) +} + +// GET is a shortcut for .Method("GET").Path(subPath) +func (w *WebService) GET(subPath string) *RouteBuilder { + return new(RouteBuilder).typeNameHandler(w.typeNameHandleFunc).servicePath(w.rootPath).Method("GET").Path(subPath) +} + +// POST is a shortcut for .Method("POST").Path(subPath) +func (w *WebService) POST(subPath string) *RouteBuilder { + return new(RouteBuilder).typeNameHandler(w.typeNameHandleFunc).servicePath(w.rootPath).Method("POST").Path(subPath) +} + +// PUT is a shortcut for .Method("PUT").Path(subPath) +func (w *WebService) PUT(subPath string) *RouteBuilder { + return new(RouteBuilder).typeNameHandler(w.typeNameHandleFunc).servicePath(w.rootPath).Method("PUT").Path(subPath) +} + +// PATCH is a shortcut for .Method("PATCH").Path(subPath) +func (w *WebService) PATCH(subPath string) *RouteBuilder { + return new(RouteBuilder).typeNameHandler(w.typeNameHandleFunc).servicePath(w.rootPath).Method("PATCH").Path(subPath) +} + +// DELETE is a shortcut for .Method("DELETE").Path(subPath) +func (w *WebService) DELETE(subPath string) *RouteBuilder { + return new(RouteBuilder).typeNameHandler(w.typeNameHandleFunc).servicePath(w.rootPath).Method("DELETE").Path(subPath) +} diff --git a/vendor/github.com/emicklei/go-restful/web_service_container.go b/vendor/github.com/emicklei/go-restful/web_service_container.go new file mode 100644 index 00000000000..c9d31b06c47 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/web_service_container.go @@ -0,0 +1,39 @@ +package restful + +// Copyright 2013 Ernest Micklei. All rights reserved. +// Use of this source code is governed by a license +// that can be found in the LICENSE file. + +import ( + "net/http" +) + +// DefaultContainer is a restful.Container that uses http.DefaultServeMux +var DefaultContainer *Container + +func init() { + DefaultContainer = NewContainer() + DefaultContainer.ServeMux = http.DefaultServeMux +} + +// If set the true then panics will not be caught to return HTTP 500. +// In that case, Route functions are responsible for handling any error situation. +// Default value is false = recover from panics. This has performance implications. +// OBSOLETE ; use restful.DefaultContainer.DoNotRecover(true) +var DoNotRecover = false + +// Add registers a new WebService add it to the DefaultContainer. +func Add(service *WebService) { + DefaultContainer.Add(service) +} + +// Filter appends a container FilterFunction from the DefaultContainer. +// These are called before dispatching a http.Request to a WebService. +func Filter(filter FilterFunction) { + DefaultContainer.Filter(filter) +} + +// RegisteredWebServices returns the collections of WebServices from the DefaultContainer +func RegisteredWebServices() []*WebService { + return DefaultContainer.RegisteredWebServices() +} diff --git a/vendor/github.com/emicklei/go-restful/web_service_test.go b/vendor/github.com/emicklei/go-restful/web_service_test.go new file mode 100644 index 00000000000..c1b756a94e1 --- /dev/null +++ b/vendor/github.com/emicklei/go-restful/web_service_test.go @@ -0,0 +1,343 @@ +package restful + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +const ( + pathGetFriends = "/get/{userId}/friends" +) + +func TestParameter(t *testing.T) { + p := &Parameter{&ParameterData{Name: "name", Description: "desc"}} + p.AllowMultiple(true) + p.DataType("int") + p.Required(true) + values := map[string]string{"a": "b"} + p.AllowableValues(values) + p.bePath() + + ws := new(WebService) + ws.Param(p) + if ws.pathParameters[0].Data().Name != "name" { + t.Error("path parameter (or name) invalid") + } +} +func TestWebService_CanCreateParameterKinds(t *testing.T) { + ws := new(WebService) + if ws.BodyParameter("b", "b").Kind() != BodyParameterKind { + t.Error("body parameter expected") + } + if ws.PathParameter("p", "p").Kind() != PathParameterKind { + t.Error("path parameter expected") + } + if ws.QueryParameter("q", "q").Kind() != QueryParameterKind { + t.Error("query parameter expected") + } +} + +func TestCapturePanic(t *testing.T) { + tearDown() + Add(newPanicingService()) + httpRequest, _ := http.NewRequest("GET", "http://here.com/fire", nil) + httpRequest.Header.Set("Accept", "*/*") + httpWriter := httptest.NewRecorder() + // override the default here + DefaultContainer.DoNotRecover(false) + DefaultContainer.dispatch(httpWriter, httpRequest) + if 500 != httpWriter.Code { + t.Error("500 expected on fire") + } +} + +func TestCapturePanicWithEncoded(t *testing.T) { + tearDown() + Add(newPanicingService()) + DefaultContainer.EnableContentEncoding(true) + httpRequest, _ := http.NewRequest("GET", "http://here.com/fire", nil) + httpRequest.Header.Set("Accept", "*/*") + httpRequest.Header.Set("Accept-Encoding", "gzip") + httpWriter := httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if 500 != httpWriter.Code { + t.Error("500 expected on fire, got", httpWriter.Code) + } +} + +func TestNotFound(t *testing.T) { + tearDown() + httpRequest, _ := http.NewRequest("GET", "http://here.com/missing", nil) + httpRequest.Header.Set("Accept", "*/*") + httpWriter := httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if 404 != httpWriter.Code { + t.Error("404 expected on missing") + } +} + +func TestMethodNotAllowed(t *testing.T) { + tearDown() + Add(newGetOnlyService()) + httpRequest, _ := http.NewRequest("POST", "http://here.com/get", nil) + httpRequest.Header.Set("Accept", "*/*") + httpWriter := httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if 405 != httpWriter.Code { + t.Error("405 expected method not allowed") + } +} + +func TestSelectedRoutePath_Issue100(t *testing.T) { + tearDown() + Add(newSelectedRouteTestingService()) + httpRequest, _ := http.NewRequest("GET", "http://here.com/get/232452/friends", nil) + httpRequest.Header.Set("Accept", "*/*") + httpWriter := httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if http.StatusOK != httpWriter.Code { + t.Error(http.StatusOK, "expected,", httpWriter.Code, "received.") + } +} + +func TestContentType415_Issue170(t *testing.T) { + tearDown() + Add(newGetOnlyJsonOnlyService()) + httpRequest, _ := http.NewRequest("GET", "http://here.com/get", nil) + httpWriter := httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if 200 != httpWriter.Code { + t.Errorf("Expected 200, got %d", httpWriter.Code) + } +} + +func TestNoContentTypePOST(t *testing.T) { + tearDown() + Add(newPostNoConsumesService()) + httpRequest, _ := http.NewRequest("POST", "http://here.com/post", nil) + httpWriter := httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if 204 != httpWriter.Code { + t.Errorf("Expected 204, got %d", httpWriter.Code) + } +} + +func TestContentType415_POST_Issue170(t *testing.T) { + tearDown() + Add(newPostOnlyJsonOnlyService()) + httpRequest, _ := http.NewRequest("POST", "http://here.com/post", nil) + httpRequest.Header.Set("Content-Type", "application/json") + httpWriter := httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if 200 != httpWriter.Code { + t.Errorf("Expected 200, got %d", httpWriter.Code) + } +} + +// go test -v -test.run TestContentType406PlainJson ...restful +func TestContentType406PlainJson(t *testing.T) { + tearDown() + TraceLogger(testLogger{t}) + Add(newGetPlainTextOrJsonService()) + httpRequest, _ := http.NewRequest("GET", "http://here.com/get", nil) + httpRequest.Header.Set("Accept", "text/plain") + httpWriter := httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if got, want := httpWriter.Code, 200; got != want { + t.Errorf("got %v, want %v", got, want) + } +} + +func TestRemoveRoute(t *testing.T) { + tearDown() + TraceLogger(testLogger{t}) + ws := newGetPlainTextOrJsonService() + Add(ws) + httpRequest, _ := http.NewRequest("GET", "http://here.com/get", nil) + httpRequest.Header.Set("Accept", "text/plain") + httpWriter := httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if got, want := httpWriter.Code, 200; got != want { + t.Errorf("got %v, want %v", got, want) + } + + // dynamic apis are disabled, should error and do nothing + if err := ws.RemoveRoute("/get", "GET"); err == nil { + t.Error("unexpected non-error") + } + + httpWriter = httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if got, want := httpWriter.Code, 200; got != want { + t.Errorf("got %v, want %v", got, want) + } + + ws.SetDynamicRoutes(true) + if err := ws.RemoveRoute("/get", "GET"); err != nil { + t.Errorf("unexpected error %v", err) + } + + httpWriter = httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if got, want := httpWriter.Code, 404; got != want { + t.Errorf("got %v, want %v", got, want) + } +} +func TestRemoveLastRoute(t *testing.T) { + tearDown() + TraceLogger(testLogger{t}) + ws := newGetPlainTextOrJsonServiceMultiRoute() + Add(ws) + httpRequest, _ := http.NewRequest("GET", "http://here.com/get", nil) + httpRequest.Header.Set("Accept", "text/plain") + httpWriter := httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if got, want := httpWriter.Code, 200; got != want { + t.Errorf("got %v, want %v", got, want) + } + + // dynamic apis are disabled, should error and do nothing + if err := ws.RemoveRoute("/get", "GET"); err == nil { + t.Error("unexpected non-error") + } + + httpWriter = httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if got, want := httpWriter.Code, 200; got != want { + t.Errorf("got %v, want %v", got, want) + } + + ws.SetDynamicRoutes(true) + if err := ws.RemoveRoute("/get", "GET"); err != nil { + t.Errorf("unexpected error %v", err) + } + + httpWriter = httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if got, want := httpWriter.Code, 404; got != want { + t.Errorf("got %v, want %v", got, want) + } +} + +// go test -v -test.run TestContentTypeOctet_Issue170 ...restful +func TestContentTypeOctet_Issue170(t *testing.T) { + tearDown() + Add(newGetConsumingOctetStreamService()) + // with content-type + httpRequest, _ := http.NewRequest("GET", "http://here.com/get", nil) + httpRequest.Header.Set("Content-Type", MIME_OCTET) + httpWriter := httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if 200 != httpWriter.Code { + t.Errorf("Expected 200, got %d", httpWriter.Code) + } + // without content-type + httpRequest, _ = http.NewRequest("GET", "http://here.com/get", nil) + httpWriter = httptest.NewRecorder() + DefaultContainer.dispatch(httpWriter, httpRequest) + if 200 != httpWriter.Code { + t.Errorf("Expected 200, got %d", httpWriter.Code) + } +} + +type exampleBody struct{} + +func TestParameterDataTypeDefaults(t *testing.T) { + tearDown() + ws := new(WebService) + route := ws.POST("/post").Reads(&exampleBody{}, "") + if route.parameters[0].data.DataType != "*restful.exampleBody" { + t.Errorf("body parameter incorrect name: %#v", route.parameters[0].data) + } +} + +func TestParameterDataTypeCustomization(t *testing.T) { + tearDown() + ws := new(WebService) + ws.TypeNameHandler(func(sample interface{}) string { + return "my.custom.type.name" + }) + route := ws.POST("/post").Reads(&exampleBody{}, "") + if route.parameters[0].data.DataType != "my.custom.type.name" { + t.Errorf("body parameter incorrect name: %#v", route.parameters[0].data) + } +} + +func newPanicingService() *WebService { + ws := new(WebService).Path("") + ws.Route(ws.GET("/fire").To(doPanic)) + return ws +} + +func newGetOnlyService() *WebService { + ws := new(WebService).Path("") + ws.Route(ws.GET("/get").To(doPanic)) + return ws +} + +func newPostOnlyJsonOnlyService() *WebService { + ws := new(WebService).Path("") + ws.Consumes("application/json") + ws.Route(ws.POST("/post").To(doNothing)) + return ws +} + +func newGetOnlyJsonOnlyService() *WebService { + ws := new(WebService).Path("") + ws.Consumes("application/json") + ws.Route(ws.GET("/get").To(doNothing)) + return ws +} + +func newGetPlainTextOrJsonService() *WebService { + ws := new(WebService).Path("") + ws.Produces("text/plain", "application/json") + ws.Route(ws.GET("/get").To(doNothing)) + return ws +} + +func newGetPlainTextOrJsonServiceMultiRoute() *WebService { + ws := new(WebService).Path("") + ws.Produces("text/plain", "application/json") + ws.Route(ws.GET("/get").To(doNothing)) + ws.Route(ws.GET("/status").To(doNothing)) + return ws +} + +func newGetConsumingOctetStreamService() *WebService { + ws := new(WebService).Path("") + ws.Consumes("application/octet-stream") + ws.Route(ws.GET("/get").To(doNothing)) + return ws +} + +func newPostNoConsumesService() *WebService { + ws := new(WebService).Path("") + ws.Route(ws.POST("/post").To(return204)) + return ws +} + +func newSelectedRouteTestingService() *WebService { + ws := new(WebService).Path("") + ws.Route(ws.GET(pathGetFriends).To(selectedRouteChecker)) + return ws +} + +func selectedRouteChecker(req *Request, resp *Response) { + if req.SelectedRoutePath() != pathGetFriends { + resp.InternalServerError() + } +} + +func doPanic(req *Request, resp *Response) { + println("lightning...") + panic("fire") +} + +func doNothing(req *Request, resp *Response) { +} + +func return204(req *Request, resp *Response) { + resp.WriteHeader(204) +} From 40fa7e22c0f246760a49d056a7f5eb15be567f9f Mon Sep 17 00:00:00 2001 From: weekface Date: Wed, 31 Oct 2018 11:05:06 +0800 Subject: [PATCH 10/12] disable cgo --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 53aaa0e2a10..b0e68960897 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ GOENV := GO15VENDOREXPERIMENT="1" CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO := $(GOENV) go -GOTEST := go test -v -cover +GOTEST := CGO_ENABLED=0 go test -v -cover LDFLAGS += -X "github.com/pingcap/tidb-operator/version.BuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')" LDFLAGS += -X "github.com/pingcap/tidb-operator/version.GitSHA=$(shell git rev-parse HEAD)" From 682d8c27b1ccffe68f5a6846991ac576ee11aeb4 Mon Sep 17 00:00:00 2001 From: weekface Date: Thu, 1 Nov 2018 14:56:27 +0800 Subject: [PATCH 11/12] address comment --- images/tidb-operator-e2e/tidb-operator-values.yaml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/images/tidb-operator-e2e/tidb-operator-values.yaml b/images/tidb-operator-e2e/tidb-operator-values.yaml index 9a1575433ef..852ef9d6a6c 100644 --- a/images/tidb-operator-e2e/tidb-operator-values.yaml +++ b/images/tidb-operator-e2e/tidb-operator-values.yaml @@ -50,10 +50,4 @@ scheduler: requests: cpu: 80m memory: 50Mi - # pd replicas - pdReplicas: 3 - # tikv replicas - tikvReplicas: 3 - hyperkubeImage: quay.io/coreos/hyperkube:v1.10.4_coreos.0 - # kubeSchedulerImage: - + kubeSchedulerImage: quay.io/coreos/hyperkube:v1.10.4_coreos.0 From 60db86d1fb7d8338aba75e346c888424c89f0614 Mon Sep 17 00:00:00 2001 From: weekface Date: Thu, 1 Nov 2018 20:10:26 +0800 Subject: [PATCH 12/12] address comment --- pkg/scheduler/predicates/ha.go | 14 ++++++------ pkg/scheduler/predicates/ha_test.go | 33 ----------------------------- 2 files changed, 8 insertions(+), 39 deletions(-) diff --git a/pkg/scheduler/predicates/ha.go b/pkg/scheduler/predicates/ha.go index 01b8dd1acc2..b6111da6039 100644 --- a/pkg/scheduler/predicates/ha.go +++ b/pkg/scheduler/predicates/ha.go @@ -46,6 +46,10 @@ func (h *ha) Name() string { return "HighAvailability" } +// 1. First, we sort all the nodes we get from kube-scheduler by how many same kind of pod it contains, +// find the nodes that have least pods. +// 2. When scheduling the first replicas pods, we must ensure no previous pods on the nodes. +// 3. For later pods, we choose the nodes that have least pods. func (h *ha) Filter(clusterName string, pod *apiv1.Pod, nodes []apiv1.Node) ([]apiv1.Node, error) { ns := pod.GetNamespace() podName := pod.GetName() @@ -102,18 +106,16 @@ func (h *ha) Filter(clusterName string, pod *apiv1.Pod, nodes []apiv1.Node) ([]a min = count } } + if ordinal < replicas && min != 0 { + return nil, fmt.Errorf("the first %d pods can't be scheduled to the same node", replicas) + } + minNodeNames := make([]string, 0) for nodeName, podNameArr := range nodeMap { if len(podNameArr) == min { minNodeNames = append(minNodeNames, nodeName) } } - if len(minNodeNames) == 0 { - return nil, fmt.Errorf("no suitable node for pod: %s/%s", ns, podName) - } - if ordinal < replicas && min != 0 { - return nil, fmt.Errorf("the first %d pods can't be scheduled to the same node", replicas) - } return getNodeFromNames(nodes, minNodeNames), nil } diff --git a/pkg/scheduler/predicates/ha_test.go b/pkg/scheduler/predicates/ha_test.go index c0a80080f9a..a7e531fe38f 100644 --- a/pkg/scheduler/predicates/ha_test.go +++ b/pkg/scheduler/predicates/ha_test.go @@ -157,17 +157,6 @@ func TestHAFilter(t *testing.T) { g.Expect(strings.Contains(err.Error(), "the first 3 pods can't be scheduled to the same node")).To(Equal(true)) }, }, - { - name: "ordinal 0 is scheduled to kube-node-3, get 0 node, ordinal 1 should be scheduled to none", - ordinal: 1, - podFn: newHAPDPod, - nodesFn: fakeZeroNode, - podListFn: podListFn(map[string][]int32{"kube-node-3": {0}}), - expectFn: func(nodes []apiv1.Node, err error) { - g.Expect(err).To(HaveOccurred()) - g.Expect(strings.Contains(err.Error(), "no suitable node for pod: default/demo-pd-1")).To(Equal(true)) - }, - }, { name: "ordinal 0 is scheduled to kube-node-2, ordinal 1 is kube-node-3, ordinal 2 should be scheduled to kube-node-1", ordinal: 2, @@ -191,17 +180,6 @@ func TestHAFilter(t *testing.T) { g.Expect(strings.Contains(err.Error(), "the first 3 pods can't be scheduled to the same node")).To(Equal(true)) }, }, - { - name: "ordinal 0 is scheduled to kube-node-1, ordinal 1 is kube-node-3, get 0 node, ordinal 2 should be scheduled to none", - ordinal: 2, - podFn: newHAPDPod, - nodesFn: fakeZeroNode, - podListFn: podListFn(map[string][]int32{"kube-node-1": {0}, "kube-node-3": {1}}), - expectFn: func(nodes []apiv1.Node, err error) { - g.Expect(err).To(HaveOccurred()) - g.Expect(strings.Contains(err.Error(), "no suitable node for pod: default/demo-pd-2")).To(Equal(true)) - }, - }, { name: "the first three oridnals get to 3 nodes, the ordinal 3 should scheduled to 1,2,3", ordinal: 3, @@ -262,17 +240,6 @@ func TestHAFilter(t *testing.T) { g.Expect(getSortedNodeNames(nodes)).To(Equal([]string{"kube-node-3"})) }, }, - { - name: "the first five oridnals get to 3 nodes, got 0 nodes, the ordinal 5 should scheduled to none", - ordinal: 5, - podFn: newHAPDPod, - nodesFn: fakeZeroNode, - podListFn: podListFn(map[string][]int32{"kube-node-2": {0}, "kube-node-3": {1, 4}, "kube-node-1": {2, 3}}), - expectFn: func(nodes []apiv1.Node, err error) { - g.Expect(err).To(HaveOccurred()) - g.Expect(strings.Contains(err.Error(), "no suitable node for pod: default/demo-pd-5")).To(Equal(true)) - }, - }, { name: "the first six oridnals get to 3 nodes, the ordinal 6 should scheduled to 1,2,3", ordinal: 6,