refactor: Perf tests refactor and optimizations (#1615)

* Allow custom memory profiling rate (via `DEBUG_PROFILE_MEMRATE` variable) Signed-off-by: Ondrej Fabry <ofabry@cisco.com> * Cleanup .gitignore files Signed-off-by: Ondrej Fabry <ofabry@cisco.com> * Improve perf tests Signed-off-by: Ondrej Fabry <ofabry@cisco.com> * Cache return values of KeyPrefix and Name methods for models Signed-off-by: Ondrej Fabry <ofabry@cisco.com> * Define configurable REST addr for perf tests Signed-off-by: Ondrej Fabry <ofabry@cisco.com>
ligato · Feb 4, 2020 · 0c8bd9f · 0c8bd9f
1 parent c69d219
commit 0c8bd9f
Show file tree

Hide file tree

Showing 7 changed files with 103 additions and 45 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,7 +2,9 @@
 *.so
 *.test
 *.out
+*.log
 
+# ignore IDE files
 .idea/*
 
 # ignore executables (extensionless files)
@@ -13,16 +15,8 @@
 !/examples/**/
 !/examples/**/*.*
 
-# test results
-/tests/perf/grpc-perf/grpc-perf
-/tests/perf/log
-/tests/perf/logresult.zip
-/tests/robot/variables/jozo_local_variables.robot
-
-# vpp directory
+# top level dirs
 /vpp/
-
-# exclude vendor
 /vendor/
-
 /.build/
+/report/
diff --git a/pkg/debug/debug.go b/pkg/debug/debug.go
@@ -20,17 +20,23 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/pkg/profile"
 )
 
-const defaultServerAddr = ":1234"
+const (
+	defaultServerAddr     = ":1234"
+	defaultMemProfileRate = 1024
+)
 
 var (
-	profileMode     = os.Getenv("DEBUG_PROFILE_MODE")
-	profilePath     = os.Getenv("DEBUG_PROFILE_PATH")
 	debugServerAddr = os.Getenv("DEBUG_SERVER_ADDR")
+
+	profileMode    = os.Getenv("DEBUG_PROFILE_MODE")
+	profilePath    = os.Getenv("DEBUG_PROFILE_PATH")
+	profileMemRate = os.Getenv("DEBUG_PROFILE_MEMRATE")
 )
 
 type Debug struct {
@@ -58,11 +64,18 @@ func (d *Debug) Stop() {
 func (d *Debug) runProfiling() {
 	var profiling func(*profile.Profile)
 
+	memRate := defaultMemProfileRate
+	if profileMemRate != "" {
+		if x, err := strconv.Atoi(profileMemRate); err == nil {
+			memRate = x
+		}
+	}
+
 	switch strings.ToLower(profileMode) {
 	case "cpu":
 		profiling = profile.CPUProfile
 	case "mem":
-		profiling = profile.MemProfile
+		profiling = profile.MemProfileRate(memRate)
 	case "mutex":
 		profiling = profile.MutexProfile
 	case "block":

diff --git a/pkg/models/model.go b/pkg/models/model.go
@@ -30,16 +30,20 @@ type KnownModel struct {
 
 	goType    reflect.Type
 	protoName string
+
+	// cache
+	keyPrefix *string
+	modelName *string
 }
 
 // Spec returns model specification for the model.
-func (m KnownModel) Spec() *Spec {
+func (m *KnownModel) Spec() *Spec {
 	spec := m.spec
 	return &spec
 }
 
-// ModelDescriptor returns descriptor for the model.
-func (m KnownModel) ModelDetail() *generic.ModelDetail {
+// ModelDetail returns descriptor for the model.
+func (m *KnownModel) ModelDetail() *generic.ModelDetail {
 	return &generic.ModelDetail{
 		Spec:      m.Spec().Proto(),
 		ProtoName: m.ProtoName(),
@@ -51,35 +55,47 @@ func (m KnownModel) ModelDetail() *generic.ModelDetail {
 }
 
 // NewInstance creates new instance value for model type.
-func (m KnownModel) NewInstance() proto.Message {
+func (m *KnownModel) NewInstance() proto.Message {
 	return reflect.New(m.goType.Elem()).Interface().(proto.Message)
 }
 
 // ProtoName returns proto message name registered with the model.
-func (m KnownModel) ProtoName() string {
+func (m *KnownModel) ProtoName() string {
 	if m.protoName == "" {
 		m.protoName = proto.MessageName(m.NewInstance())
 	}
 	return m.protoName
 }
 
 // NameTemplate returns name template for the model.
-func (m KnownModel) NameTemplate() string {
+func (m *KnownModel) NameTemplate() string {
 	return m.nameTemplate
 }
 
 // GoType returns go type for the model.
-func (m KnownModel) GoType() string {
+func (m *KnownModel) GoType() string {
 	return m.goType.String()
 }
 
-// Path returns path for the model.
-func (m KnownModel) Name() string {
-	return m.spec.ModelName()
+// Name returns name for the model.
+func (m *KnownModel) Name() string {
+	if m.modelName == nil {
+		modelName := m.spec.ModelName()
+		m.modelName = &modelName
+	}
+	return *m.modelName
 }
 
 // KeyPrefix returns key prefix for the model.
-func (m KnownModel) KeyPrefix() string {
+func (m *KnownModel) KeyPrefix() string {
+	if m.keyPrefix == nil {
+		keyPrefix := m.getKeyPrefix()
+		m.keyPrefix = &keyPrefix
+	}
+	return *m.keyPrefix
+}
+
+func (m *KnownModel) getKeyPrefix() string {
 	keyPrefix := m.spec.KeyPrefix()
 	if m.nameFunc == nil {
 		keyPrefix = strings.TrimSuffix(keyPrefix, "/")
@@ -89,7 +105,7 @@ func (m KnownModel) KeyPrefix() string {
 
 // ParseKey parses the given key and returns item name
 // or returns empty name and valid as false if the key is not valid.
-func (m KnownModel) ParseKey(key string) (name string, valid bool) {
+func (m *KnownModel) ParseKey(key string) (name string, valid bool) {
 	name = strings.TrimPrefix(key, m.KeyPrefix())
 	if name == key || (name == "" && m.nameFunc != nil) {
 		name = strings.TrimPrefix(key, m.Name())
@@ -104,20 +120,20 @@ func (m KnownModel) ParseKey(key string) (name string, valid bool) {
 }
 
 // IsKeyValid returns true if given key is valid for this model.
-func (m KnownModel) IsKeyValid(key string) bool {
+func (m *KnownModel) IsKeyValid(key string) bool {
 	_, valid := m.ParseKey(key)
 	return valid
 }
 
 // StripKeyPrefix returns key with prefix stripped.
-func (m KnownModel) StripKeyPrefix(key string) string {
+func (m *KnownModel) StripKeyPrefix(key string) string {
 	if name, valid := m.ParseKey(key); valid {
 		return name
 	}
 	return key
 }
 
-func (m KnownModel) instanceName(x proto.Message) (string, error) {
+func (m *KnownModel) instanceName(x proto.Message) (string, error) {
 	if m.nameFunc == nil {
 		return "", nil
 	}

diff --git a/tests/.gitignore b/tests/.gitignore
@@ -0,0 +1,8 @@
+log/
+logresult.zip
+
+reports/
+
+/robot/variables/jozo_local_variables.robot
+
+/perf/grpc-perf/grpc-perf
diff --git a/tests/perf/.gitignore b/tests/perf/.gitignore
diff --git a/tests/perf/grpc-perf/main.go b/tests/perf/grpc-perf/main.go
@@ -29,6 +29,7 @@ import (
 	"github.com/ligato/cn-infra/infra"
 	"github.com/ligato/cn-infra/logging"
 	"github.com/namsral/flag"
+	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
 
 	"go.ligato.io/vpp-agent/v3/proto/ligato/configurator"
@@ -80,7 +81,8 @@ func main() {
 
 // GRPCStressPlugin makes use of the remoteclient to locally CRUD ipsec tunnels and routes.
 type GRPCStressPlugin struct {
-	infra.PluginDeps
+	infra.PluginName
+	Log *logrus.Logger
 
 	conns []*grpc.ClientConn
 
@@ -90,14 +92,20 @@ type GRPCStressPlugin struct {
 func NewGRPCStressPlugin() *GRPCStressPlugin {
 	p := &GRPCStressPlugin{}
 	p.SetName("grpc-stress-test-client")
-	p.Setup()
+	p.Log = logrus.New()
+	p.Log.SetFormatter(&logrus.TextFormatter{
+		ForceColors:               true,
+		EnvironmentOverrideColors: true,
+	})
 	return p
 }
 
-// Init initializes  plugin.
 func (p *GRPCStressPlugin) Init() error {
 	return nil
 }
+func (p *GRPCStressPlugin) Close() error {
+	return nil
+}
 
 // Dialer for unix domain socket
 func dialer(socket, address string, timeoutVal time.Duration) func(string, time.Duration) (net.Conn, error) {
@@ -216,9 +224,16 @@ func (p *GRPCStressPlugin) runAllClients() {
 
 	p.Log.Debugf("Waiting for clients..")
 	p.wg.Wait()
-
-	took := time.Since(t).Round(time.Microsecond * 100)
-	p.Log.Infof("All clients done, took: %v", took)
+	took := time.Since(t)
+	perSec := float64(*numTunnels) / took.Seconds()
+
+	p.Log.Infof("All clients done!")
+	p.Log.Infof("----------------------------------------")
+	p.Log.Infof(" -> Took: %.3fs", took.Seconds())
+	p.Log.Infof(" -> Clients: %d", *numClients)
+	p.Log.Infof(" -> Requests: %d", *numTunnels)
+	p.Log.Infof(" -> PERFORMANCE: %.1f req/sec", perSec)
+	p.Log.Infof("----------------------------------------")
 
 	for i := 0; i < *numClients; i++ {
 		if err := p.conns[i].Close(); err != nil {

diff --git a/tests/perf/perf_test.sh b/tests/perf/perf_test.sh
@@ -28,13 +28,15 @@ agent_info="${REPORT_DIR}/agent-info.txt"
 cpuprof="${REPORT_DIR}/cpu.pprof"
 memprof="${REPORT_DIR}/mem.pprof"
 
+rest_addr="${REST_ADDR:-http://127.0.0.1:9191}"
+
 # -------
 #  test
 # -------
 
 function run_test() {
 	# create report directory
-	rm -vrf ${REPORT_DIR}/*
+	rm -vrf ${REPORT_DIR}/* 2>/dev/null
 	mkdir --mode=777 -p ${REPORT_DIR}
 
 	perftest $* 2>&1 | tee $log_report
@@ -45,8 +47,9 @@ function perftest() {
 	local requests="$2"
 
 	echo "================================================================================"
-	echo " PERF-TEST: $perftest - ${requests} requests"
-	echo "  -> ${REPORT_DIR}"
+	echo " PERF TEST: $perftest"
+	echo "  - num requests: ${requests}"
+	echo "  - report dir: ${REPORT_DIR}"
 	echo "================================================================================"
 
 	prepare_test
@@ -116,8 +119,14 @@ function perftest() {
 	stop_vpp
 
 	echo -n "-> processing profiles.. "
-	[ -r "$cpuprof" ] && go tool pprof -dot "$cpuprof" | dot -Tsvg -o "$REPORT_DIR/cpu-profile.svg"
-	[ -r "$memprof" ] && go tool pprof -alloc_space -dot "$memprof" | dot -Tsvg -o "$REPORT_DIR/mem-profile.svg"
+	set +e
+	if [ -r "$cpuprof" ]; then
+		go tool pprof -dot "$cpuprof" | dot -Tsvg -o "$REPORT_DIR/cpu-profile.svg"
+	fi
+	if [ -r "$memprof" ]; then
+		go tool pprof -alloc_space -dot "$memprof" | dot -Tsvg -o "$REPORT_DIR/mem-profile.svg"
+	fi
+	set -e
 
 	echo
 }
@@ -129,7 +138,7 @@ function prepare_test() {
 
 	#[[ -e "./$_test" ]] || {
 		echo "-> compiling test client $_test.."
-		go build -o "$_test_client/$_test" -v "$_test_client"
+		go build -o "$_test_client/$_test" "$_test_client"
 	#}
 }
 
@@ -235,7 +244,7 @@ function vppcli() {
 #  vpp-agent
 # -------------
 
-wait_agent_boot=5
+wait_agent_boot=10
 
 function start_agent() {
 	local _agent="$(which vpp-agent)"
@@ -247,7 +256,11 @@ function start_agent() {
 	$_agent > "$log_agent" 2>&1 &
 	pid_agent="$!"
 	timeout "${wait_agent_boot}" grep -q "Agent started" <(tail -qF $log_agent) || {
-		fail "timeout!"
+		echo "AGENT LOG:"
+		echo "---"
+		tail "$log_agent"
+		echo "---"
+		fail "TIMEOUT!"
 	}
 	echo "ok! (PID:${pid_agent})"
 }
@@ -287,7 +300,7 @@ function check_agent() {
 }
 
 function agentrest() {
-	local url="http://localhost:9191/$1"
+	local url="$rest_addr/$1"
 
 	echo "----------------------------------------------------"
 	echo "GET $url"