Support grpc client

.travis.yml

@@ -19,6 +19,10 @@ before_install:
   - wget http://us.archive.ubuntu.com/ubuntu/pool/universe/w/wrk/wrk_4.0.1-2_amd64.deb
   - sudo dpkg -i wrk_4.0.1-2_amd64.deb
   - rm wrk_4.0.1-2_amd64.deb
+  - PROTOC_ZIP=protoc-3.7.1-linux-x86_64.zip
+  - curl -OL https://github.com/google/protobuf/releases/download/v3.7.1/$PROTOC_ZIP
+  - sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc
+  - rm -f $PROTOC_ZIP
 install:
   - make jenkins-install
 script:

Makefile

@@ -23,6 +23,8 @@ EXAMPLE_SERVICES = $(sort $(dir $(wildcard $(EXAMPLE_SERVICES_DIR)*/)))
 GOIMPORTS = "$(PWD)/vendor/golang.org/x/tools/cmd/goimports"
 GOBINDATA = "$(PWD)/vendor/github.com/jteeuwen/go-bindata/go-bindata"
 GOMOCK = "$(PWD)/vendor/github.com/golang/mock/mockgen"
+GOGOSLICK = "$(PWD)/vendor/github.com/gogo/protobuf/protoc-gen-gogoslick"
+YARPCGO = "$(PWD)/vendor/go.uber.org/yarpc/encoding/protobuf/protoc-gen-yarpc-go"
 
 .PHONY: install
 install:
@@ -35,6 +37,8 @@ install:
 	go build -o $(GOIMPORTS)/goimports ./vendor/golang.org/x/tools/cmd/goimports/
 	go build -o $(GOBINDATA)/go-bindata ./vendor/github.com/jteeuwen/go-bindata/go-bindata/
 	go build -o $(GOMOCK)/mockgen ./vendor/github.com/golang/mock/mockgen/
+	go build -o $(GOGOSLICK)/protoc-gen-gogoslick ./vendor/github.com/gogo/protobuf/protoc-gen-gogoslick/
+	go build -o $(YARPCGO)/protoc-gen-yarpc-go ./vendor/go.uber.org/yarpc/encoding/protobuf/protoc-gen-yarpc-go/
 
 .PHONY: check-licence
 check-licence:
@@ -112,7 +116,7 @@ generate:
 	@./node_modules/.bin/uber-licence --file "production.gen.go" --dir "config" > /dev/null
 	@$(GOBINDATA)/go-bindata -pkg templates -nocompress -modtime 1 -prefix codegen/templates -o codegen/template_bundle/template_files.go codegen/templates/...
 	@gofmt -w -e -s "codegen/template_bundle/template_files.go"
-	@PATH=$(GOIMPORTS):$(GOMOCK):$(PATH) bash ./scripts/generate.sh
+	@PATH=$(GOGOSLICK):$(YARPCGO):$(GOIMPORTS):$(GOMOCK):$(PATH) bash ./scripts/generate.sh
 
 .PHONY: check-generate
 check-generate:

codegen/module_system.go

@@ -23,6 +23,7 @@ package codegen
 import (
 	"encoding/json"
 	"net/textproto"
+	"path"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -304,6 +305,16 @@ func NewDefaultModuleSystem(
 		)
 	}
 
+	if err := system.RegisterClassType("client", "grpc", &YarpcClientGenerator{
+		templates:     tmpl,
+		packageHelper: h,
+	}); err != nil {
+		return nil, errors.Wrapf(
+			err,
+			"Error registering grpc client class type",
+		)
+	}
+
 	if err := system.RegisterClass(ModuleClass{
 		Name:       "middleware",
 		NamePlural: "middlewares",
@@ -772,6 +783,106 @@ func (g *CustomClientGenerator) Generate(
 	}, nil
 }
 
+/*
+ * yarpc client generator
+ */
+
+// YarpcClientGenerator generates grpc clients.
+type YarpcClientGenerator struct {
+	templates     *Template
+	packageHelper *PackageHelper
+}
+
+// ComputeSpec returns the spec for a yarpc client
+func (g *YarpcClientGenerator) ComputeSpec(
+	instance *ModuleInstance,
+) (interface{}, error) {
+	return (*ClientSpec)(nil), nil
+}
+
+// Generate returns the yarpc client build result, which contains the files and
+// the generated client spec
+func (g *YarpcClientGenerator) Generate(
+	instance *ModuleInstance,
+) (*BuildResult, error) {
+	clientConfig := &ClassConfig{}
+	if err := yaml.Unmarshal(instance.YAMLFileRaw, &clientConfig); err != nil {
+		return nil, errors.Wrapf(
+			err,
+			"Error reading yarpc client %q YAML config",
+			instance.InstanceName,
+		)
+	}
+
+	data := &struct {
+		Instance *ModuleInstance
+		GenPkg   string
+	}{
+		Instance: instance,
+	}
+
+	v, ok := clientConfig.Config["protoFile"]
+	if !ok {
+		return nil, errors.Errorf(
+			"Missing \"protoFile\" field in %q YAML config",
+			instance.InstanceName,
+		)
+	}
+	protoFile := v.(string)
+	parts := strings.Split(protoFile, "/")
+	genDir := strings.Join(parts[0:len(parts)-1], "/")
+
+	data.GenPkg = path.Join(
+		g.packageHelper.GenCodePackage(),
+		genDir,
+	)
+
+	client, err := g.templates.ExecTemplate(
+		"yarpc_client.tmpl",
+		data,
+		g.packageHelper,
+	)
+	if err != nil {
+		return nil, errors.Wrapf(
+			err,
+			"Error executing YARPC client template for %q",
+			instance.InstanceName,
+		)
+	}
+
+	// When it is possible to generate structs for all module types, the
+	// module system will do this transparently. For now we are opting in
+	// on a per-generator basis.
+	dependencies, err := GenerateDependencyStruct(
+		instance,
+		g.packageHelper,
+		g.templates,
+	)
+	if err != nil {
+		return nil, errors.Wrapf(
+			err,
+			"Error generating dependencies struct for %q %q",
+			instance.ClassName,
+			instance.InstanceName,
+		)
+	}
+
+	baseName := filepath.Base(instance.Directory)
+	clientFilePath := baseName + ".go"
+
+	files := map[string][]byte{
+		clientFilePath: client,
+	}
+
+	if dependencies != nil {
+		files["module/dependencies.go"] = dependencies
+	}
+	return &BuildResult{
+		Files: files,
+		Spec:  (*ClientSpec)(nil),
+	}, nil
+}
+
 /*
  * Endpoint Generator
  */

codegen/package.go

@@ -239,6 +239,11 @@ func (p PackageHelper) DefaultMiddlewareSpecs() map[string]*MiddlewareSpec {
 	return p.defaultMiddlewareSpecs
 }
 
+// GenCodePackage returns the file path to the idl generated code folder
+func (p PackageHelper) GenCodePackage() string {
+	return p.genCodePackage
+}
+
 // TypeImportPath returns the Go import path for types defined in a thrift file.
 func (p PackageHelper) TypeImportPath(thrift string) (string, error) {
 	if !strings.HasSuffix(thrift, ".thrift") {

codegen/runner/pre-steps.sh

@@ -2,16 +2,16 @@
 set -e
 set -o pipefail
 
-if [ -z "$1" ]; then
+if [[ -z "$1" ]]; then
 	echo "build dir argument (\$1) is missing"
 	exit 1
 fi
 
-if [ -z "$2" ]; then
+if [[ -z "$2" ]]; then
 	echo "config dir argument (\$2) is missing"
 	exit 1
 fi
-if [ -z "$3" ]; then
+if [[ -z "$3" ]]; then
 	echo "annotation prefix argument (\$3) is missing"
 	exit 1
 fi
@@ -23,7 +23,7 @@ ANNOPREFIX="$3"
 
 IDL_DIR="${IDL_DIR:-$CONFIG_DIR/idl}"
 
-if [ -z "$4" ]; then
+if [[ -z "$4" ]]; then
 	THRIFTRW_SRCS="$(find "$IDL_DIR" -name '*.thrift')"
 else
 	THRIFTRW_SRCS="$4"
@@ -40,7 +40,7 @@ RESOLVE_THRIFT_BINARY="$DIRNAME/../../scripts/resolve_thrift/resolve_thrift"
 RESOLVE_I64_FILE="$DIRNAME/../../scripts/resolve_i64/main.go"
 RESOLVE_I64_BINARY="$DIRNAME/../../scripts/resolve_i64/resolve_i64"
 
-if [ -d "$DIRNAME/../../vendor" ]; then
+if [[ -d "$DIRNAME/../../vendor" ]]; then
 	THRIFTRW_RAW_DIR="$DIRNAME/../../vendor/go.uber.org/thriftrw"
 	THRIFTRW_DIR="$(cd "$THRIFTRW_RAW_DIR";pwd)"
 	THRIFTRW_MAIN_FILE="$THRIFTRW_DIR/main.go"
@@ -67,7 +67,45 @@ for tfile in ${THRIFTRW_SRCS}; do
 		--no-embed-idl \
 		--thrift-root="$IDL_DIR" "$tfile"
 done
-gofmt -w "$BUILD_DIR/gen-code/"
+
+echo "Generating Go code from Proto files"
+ABS_IDL_DIR="$(cd "$IDL_DIR" && pwd)"
+ABS_GENCODE_DIR="$(cd "$BUILD_DIR" && pwd)/$(basename "$BUILD_DIR/gen-code")"
+config_files=$(find "$CONFIG_DIR" -name "*-config.json" -o -name "*-config.yaml" | sort)
+found_protos=""
+for config_file in ${config_files}; do
+	if [[ ${config_file} == "./vendor"* ]]; then
+		continue
+	fi
+
+	processor="$YQ"
+	if [[ ${config_file} == *.json ]]; then
+		processor="jq"
+	fi
+
+	dir=$(dirname "$config_file")
+	yaml_files=$(find "$dir" -name "*.json" -o -name "*.yaml")
+	for yaml_file in ${yaml_files}; do
+		processor="$YQ"
+		if [[ ${yaml_file} == *.json ]]; then
+			processor="jq"
+		fi
+
+		proto_file=$(${processor} -r '.. | .protoFile? | select(strings | endswith(".proto"))' "$yaml_file")
+		if [[ ! -z ${proto_file} ]] && [[ ${found_protos} != *${proto_file}* ]]; then
+			found_protos+=" $proto_file"
+			proto_dir=$(dirname "$proto_file")
+			proto_path="$ABS_IDL_DIR/$proto_dir"
+			gencode_path="$ABS_GENCODE_DIR/$proto_dir"
+			mkdir -p "$gencode_path"
+			proto_file="$ABS_IDL_DIR/$proto_file"
+			protoc --proto_path="$proto_path" --gogoslick_out="$gencode_path" \
+                   --yarpc-go_out="$gencode_path" "$proto_file"
+		fi
+	done
+done
+
+gofmt -w -s "$BUILD_DIR/gen-code/"
 
 end=$(date +%s)
 runtime=$((end-start))
@@ -82,32 +120,40 @@ go build -o "$RESOLVE_THRIFT_BINARY" "$RESOLVE_THRIFT_FILE"
 go build -o "$RESOLVE_I64_BINARY" "$RESOLVE_I64_FILE"
 
 # find the modules that actually need JSON (un)marshallers
-ABS_IDL_DIR="$(cd "$IDL_DIR" && pwd)"
-ABS_GENCODE_DIR="$(cd "$BUILD_DIR" && pwd)/$(basename "$BUILD_DIR/gen-code")"
 target_dirs=""
 found_thrifts=""
-config_files=$(find "$CONFIG_DIR" -name "*-config.json" -o -name "*-config.yaml" | sort)
+proto_path="$IDL_DIR"
 for config_file in ${config_files}; do
 	if [[ ${config_file} == "./vendor"* ]]; then
 		continue
 	fi
 
 	processor="$YQ"
-	if [[ $config_file == *.json ]]; then
+	if [[ ${config_file} == *.json ]]; then
 		processor="jq"
 	fi
 
-	module_type=$($processor -r .type "$config_file")
+	module_type=$(${processor} -r .type "$config_file")
 	[[ ${module_type} != *"http"* ]] && continue
 	dir=$(dirname "$config_file")
 	yaml_files=$(find "$dir" -name "*.json" -o -name "*.yaml")
 	for yaml_file in ${yaml_files}; do
 		processor="$YQ"
-		if [[ $yaml_file == *.json ]]; then
+		if [[ ${yaml_file} == *.json ]]; then
 			processor="jq"
 		fi
 
-		thrift_file=$($processor -r '.. | .thriftFile? | select(strings | endswith(".thrift"))' "$yaml_file")
+		thrift_file=$(${processor} -r '.. | .thriftFile? | select(strings | endswith(".thrift"))' "$yaml_file")
+
+		# process .proto files
+		proto_file=$(${processor} -r '.. | .protoFile? | select(strings | endswith(".proto"))' "$yaml_file")
+		if [[ ! -z ${proto_file} ]] && [[ ${found_protos} != *${proto_file}* ]]; then
+			found_protos+=" $proto_file"
+			proto_file="$IDL_DIR/$proto_file"
+			protoc --proto_path=
+
+		fi
+
 		[[ -z ${thrift_file} ]] && continue
 		[[ ${found_thrifts} == *${thrift_file}* ]] && continue
 		found_thrifts+=" $thrift_file"