chore: update docs (#120)

* refactor: remove unnecessary field in grpc proto

* docs: update docs

* chore: fix docs

* chore: fix docs

* chore: update proto

* chore: increase coverage

* fix(compass): apply some suggested changes

Makefile

@@ -1,7 +1,7 @@
 NAME="github.com/odpf/compass"
 VERSION=$(shell git describe --always --tags 2>/dev/null)
 COVERFILE="/tmp/compass.coverprofile"
-PROTON_COMMIT := "2481c008a1eb2525eca058b0729abc036ddcbe6a"
+PROTON_COMMIT := "efc71a54e643624f2f809bac8c095c069576c4dd"
 
 .PHONY: all build test clean install proto
 
@@ -40,6 +40,7 @@ proto: ## Generate the protobuf files
 install: ## install required dependencies
 	@echo "> installing dependencies"
 	go mod tidy
+	go get github.com/vektra/mockery/v2@v2.10.4
 	go get google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1
 	go get google.golang.org/protobuf/proto@v1.27.1
 	go get google.golang.org/grpc@v1.45.0

README.md

@@ -8,6 +8,12 @@
 
 Compass is a search and discovery engine built for querying application deployments, datasets and meta resources. It can also optionally track data flow relationships between these resources and allow the user to view a representation of the data flow graph.
 
+```
+Notes
+---
+Compass was previously called 'Columbus'. We we were migrating to the new name to accomodate our current and future use cases considering this platform does not only about metadata discovery and lineage but also revolve around metadata management. 
+```
+
 <p align="center"><img src="./docs/assets/overview.svg" /></p>
 
 ## Key Features
@@ -24,7 +30,7 @@ Discover why users choose Compass as their main data discovery and lineage servi
 
 Explore the following resources to get started with Compass:
 
-* [Guides](docs/guides) provides guidance on ingesting and queying metadata from Compass.
+* [Guides](docs/guides) provides guidance on ingesting and querying metadata from Compass.
 * [Concepts](docs/concepts) describes all important Compass concepts.
 * [Reference](docs/reference) contains details about configurations, metrics and other aspects of Compass.
 * [Contribute](docs/contribute/contribution.md) contains resources for anyone who wants to contribute to Compass.
@@ -107,4 +113,4 @@ To help you get your feet wet and get you familiar with our contribution process
 This project exists thanks to all the [contributors](https://github.com/odpf/compass/graphs/contributors).
 
 ## License
-Compass is [Apache 2.0](LICENSE) licensed.
+Compass is [Apache 2.0](LICENSE) licensed.

api/api.go

@@ -53,7 +53,6 @@ func NewHandlers(logger log.Logger, deps *Dependencies) *Handlers {
 }
 
 func NewHTTPHandlers(deps *Dependencies) *httpapi.Handler {
-
 	recordHandler := handlers.NewRecordHandler(
 		deps.Logger,
 		deps.TypeRepository,
@@ -78,13 +77,18 @@ func NewGRPCHandler(l log.Logger, deps *Dependencies) *v1beta1.Handler {
 		TagTemplateService:   deps.TagTemplateService,
 		DiscoveryRepository:  deps.DiscoveryRepository,
 
-		//deprecated
+		// deprecated
 		TypeRepository:   deps.TypeRepository,
 		DiscoveryService: deps.DiscoveryService,
 	}
 }
 
-func RegisterHTTPRoutes(cfg Config, mux *runtime.ServeMux, deps *Dependencies, handlerCollection *httpapi.Handler) error {
+func RegisterHTTPRoutes(
+	cfg Config,
+	mux *runtime.ServeMux,
+	deps *Dependencies,
+	handlerCollection *httpapi.Handler,
+) error {
 	if err := mux.HandlePath(http.MethodGet, "/ping", runtime.HandlerFunc(func(w http.ResponseWriter, r *http.Request, pathParams map[string]string) {
 		fmt.Fprintf(w, "pong")
 	})); err != nil {

api/v1beta1/search.go

@@ -56,12 +56,7 @@ func (h *Handler) SuggestAssets(ctx context.Context, req *compassv1beta1.Suggest
 	}
 
 	cfg := discovery.SearchConfig{
-		Text:          text,
-		MaxResults:    int(req.GetSize()),
-		Filters:       filterConfigFromValues(req.GetFilter()),
-		RankBy:        req.GetRankby(),
-		Queries:       queryConfigFromValues(req.GetQuery()),
-		TypeWhiteList: parseTypeWhiteList(req.GetFilter()),
+		Text: text,
 	}
 
 	suggestions, err := h.DiscoveryService.Suggest(ctx, cfg)
@@ -91,14 +86,6 @@ func filterConfigFromValues(fltMap map[string]string) map[string][]string {
 	return filter
 }
 
-func queryConfigFromValues(queryMap map[string]string) map[string]string {
-	var query = make(map[string]string)
-	if len(queryMap) > 0 {
-		query = queryMap
-	}
-	return query
-}
-
 func parseTypeWhiteList(fltMap map[string]string) (types []string) {
 	if val, ok := fltMap[whiteListQueryParamKey]; ok {
 		types = append(types, strings.Split(val, ",")...)

api/v1beta1/search_test.go

@@ -258,43 +258,12 @@ func TestSuggest(t *testing.T) {
 			},
 			Setup: func(ctx context.Context, drs *mocks.DiscoveryRecordSearcher) {
 				cfg := discovery.SearchConfig{
-					Text:    "test",
-					Filters: map[string][]string{},
-					Queries: make(map[string]string),
+					Text: "test",
 				}
 				drs.EXPECT().Suggest(ctx, cfg).Return([]string{}, fmt.Errorf("service unavailable"))
 			},
 			ExpectStatus: codes.Internal,
 		},
-		{
-			Description: "should pass filter to search config format",
-			Request: &compassv1beta1.SuggestAssetsRequest{
-				Text: "resource",
-				Query: map[string]string{
-					"description": "this is my dashboard",
-				},
-				Filter: map[string]string{
-					"data.landscape": "th",
-					"type":           "topic",
-					"service":        "kafka,rabbitmq",
-				},
-			},
-			Setup: func(ctx context.Context, drs *mocks.DiscoveryRecordSearcher) {
-
-				cfg := discovery.SearchConfig{
-					Text:          "resource",
-					TypeWhiteList: []string{"topic"},
-					Filters: map[string][]string{
-						"service":        {"kafka", "rabbitmq"},
-						"data.landscape": {"th"},
-					},
-					Queries: map[string]string{
-						"description": "this is my dashboard",
-					},
-				}
-				drs.EXPECT().Suggest(ctx, cfg).Return([]string{}, nil)
-			},
-		},
 		{
 			Description: "should return suggestions",
 			Request: &compassv1beta1.SuggestAssetsRequest{
@@ -303,9 +272,7 @@ func TestSuggest(t *testing.T) {
 			Setup: func(ctx context.Context, drs *mocks.DiscoveryRecordSearcher) {
 
 				cfg := discovery.SearchConfig{
-					Text:    "test",
-					Filters: make(map[string][]string),
-					Queries: make(map[string]string),
+					Text: "test",
 				}
 				response := []string{
 					"test",

asset/type_test.go

@@ -0,0 +1,46 @@
+package asset
+
+import "testing"
+
+func TestTypeString(t *testing.T) {
+	stringVal := TypeDashboard.String()
+	if stringVal != "dashboard" {
+		t.Fatalf("type dashboard converted to %s instead of 'dashboard'", stringVal)
+	}
+	stringVal = TypeJob.String()
+	if stringVal != "job" {
+		t.Fatalf("type job converted to %s instead of 'job'", stringVal)
+	}
+	stringVal = TypeTable.String()
+	if stringVal != "table" {
+		t.Fatalf("type table converted to %s instead of 'table'", stringVal)
+	}
+	stringVal = TypeTopic.String()
+	if stringVal != "topic" {
+		t.Fatalf("type topic converted to %s instead of 'topic'", stringVal)
+	}
+}
+
+func TestTypeIsValid(t *testing.T) {
+	aType := Type("dashboard")
+	if !aType.IsValid() {
+		t.Fatalf("type %s is not valid", aType)
+	}
+	aType = Type("job")
+	if !aType.IsValid() {
+		t.Fatalf("type %s is not valid", aType)
+	}
+	aType = Type("table")
+	if !aType.IsValid() {
+		t.Fatalf("type %s is not valid", aType)
+	}
+	aType = Type("topic")
+	if !aType.IsValid() {
+		t.Fatalf("type %s is not valid", aType)
+	}
+
+	aType = Type("random")
+	if aType.IsValid() {
+		t.Fatalf("type %s should not be valid", aType)
+	}
+}

asset/version_test.go

@@ -0,0 +1,55 @@
+package asset
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseVersionSemver(t *testing.T) {
+	t.Run("parse invalid version will return non nil error", func(t *testing.T) {
+		v := "xx"
+		sv, err := ParseVersion(v)
+		assert.Error(t, err)
+		assert.Nil(t, sv)
+	})
+
+	t.Run("parse valid version will return nil error", func(t *testing.T) {
+		v := "1.0"
+		sv, err := ParseVersion(v)
+		assert.Nil(t, err)
+		assert.Equal(t, sv.Major(), uint64(1))
+		assert.Equal(t, sv.Minor(), uint64(0))
+	})
+
+	t.Run("parse valid version with prefix 'v' will return nil error", func(t *testing.T) {
+		v := "v1.0"
+		sv, err := ParseVersion(v)
+		assert.Nil(t, err)
+		assert.Equal(t, sv.Major(), uint64(1))
+		assert.Equal(t, sv.Minor(), uint64(0))
+	})
+}
+
+func TestIncreaseMinorVersion(t *testing.T) {
+	t.Run("increase minor version of invalid version will return non nil error", func(t *testing.T) {
+		v := "xx"
+		sv, err := IncreaseMinorVersion(v)
+		assert.Error(t, err)
+		assert.Empty(t, sv)
+	})
+
+	t.Run("increase minor version of valid version will return nil error", func(t *testing.T) {
+		v := "1.0"
+		sv, err := IncreaseMinorVersion(v)
+		assert.Nil(t, err)
+		assert.Equal(t, "1.1", sv)
+	})
+
+	t.Run("increase minor version of valid version with prefix 'v' will return nil error", func(t *testing.T) {
+		v := "v1.0"
+		sv, err := IncreaseMinorVersion(v)
+		assert.Nil(t, err)
+		assert.Equal(t, "1.1", sv)
+	})
+}

config.yaml.example

@@ -1,5 +1,5 @@
 SERVER_HOST: localhost
-SERVER_PORT: 3000
+SERVER_PORT: 8080
 ELASTICSEARCH_BROKERS: http://localhost:9200
 STATSD_ENABLED: false
 STATSD_ADDRESS: 127.0.0.1:8125

docker-compose.yaml

@@ -5,6 +5,8 @@ services:
     image: 'docker.elastic.co/elasticsearch/elasticsearch:7.6.1'
     environment:
       discovery.type: single-node
+    networks:
+      - storage
   postgres:
     ports:
       - 5432:5432
@@ -13,4 +15,7 @@ services:
       POSTGRES_USER: compass
       POSTGRES_PASSWORD: compass_password
       POSTGRES_DB: compass
-      discovery.type: single-node
+    networks:
+      - storage
+networks: 
+  storage:

docs/concepts/architecture.md

@@ -1,16 +1,22 @@
 # Architecture
 
-Compass' architecture is pretty simple. It serves HTTP server with Elasticsearch as its main persistent storage.
+Compass' architecture is pretty simple. It has a client-server architecture backed by PostgreSQL as a main storage and Elasticsearch as a secondary storage and provides HTTP & gRPC interface to interact with.
 
-![Compass Architecture](../assets/architecture.jpg)
+![Compass Architecture](../assets/architecture.png)
 
 ## System Design
 ### Components
 
-#### HTTP Server
+#### gRPC Server
 
-* HTTP server is the main and only interface to interact with Compass using RESTful pattern.
+* gRPC server is the main interface to interact with Compass.
+* The protobuf file to define the interface is centralized in [odpf/proton](https://github.com/odpf/proton/tree/main/odpf/compass/v1beta1)
+#### gRPC-gateway Server
 
+* gRPC-gateway server transcodes HTTP call to gRPC call and allows client to interact with Compass using RESTful HTTP request.
+#### PostgreSQL
+
+* Compass uses PostgreSQL as it is main storage for storing all of its metadata.
 #### Elasticsearch
 
-* Compass uses Elasticsearch as it is main storage for storing all of its metadata.
+* Compass uses Elasticsearch as it is secondary storage to power search of metadata.