Merge pull request #30 from shortedapp/feature/cloudrun

utilise gogo for stdduration
shortedapp · Sep 29, 2020 · 5fd3972 · 5fd3972
2 parents 06fded6 + 61b3b59
commit 5fd3972
Show file tree

Hide file tree

Showing 12 changed files with 5,495 additions and 1,730 deletions.
diff --git a/services/watcher/cmd/cli/main.go b/services/watcher/cmd/cli/main.go
@@ -5,10 +5,10 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/shortedapp/shorted/services/watcher/internal/service/watcher"
 	"github.com/shortedapp/shorted/services/watcher/pkg/config"
-
-	"github.com/shortedapp/shorted/services/watcher/pkg/watcher"
 	"go.uber.org/zap"
+	v1 "github.com/shortedapp/shorted/shortedapis/pkg/watcher/v1"
 )
 
 var (
@@ -20,13 +20,21 @@ func main() {
 		ProjectId:      os.Getenv("PROJECT_ID"),
 		LoggingEncoder: os.Getenv("LOGGING_ENCODER"),
 	}
-	
+
 	logger = zap.S().With("watcher", "cmd")
 	ctx := context.Background()
 	w, err := watcher.New(ctx, cfg)
 	if err != nil {
 		panic(fmt.Errorf("failed initialising watch: %v", err))
 	}
-	w.Discover()
+	w.CreateWatcher(v1.CreateWatcherRequest{
+		Name: "test",
+		Source: &v1.Source{
+			Url: "https://asic.gov.au",
+			Adapter: "asic",
+			Format: v1.DocumentFormat_CSV,
+			Interval: "60m",
+		},
+	})
 	defer zap.L().Sync()
 }
diff --git a/services/watcher/cmd/run/main.go b/services/watcher/cmd/run/main.go
@@ -8,9 +8,9 @@ import (
 	"strings"
 
 	"github.com/grpc-ecosystem/grpc-gateway/runtime"
+	"github.com/shortedapp/shorted/services/watcher/internal/service/watcher"
 	"github.com/shortedapp/shorted/services/watcher/pkg/config"
 	"github.com/shortedapp/shorted/services/watcher/pkg/log"
-	"github.com/shortedapp/shorted/services/watcher/internal/service/watcher"
 	v1 "github.com/shortedapp/shorted/shortedapis/pkg/watcher/v1"
 	"go.uber.org/zap"
 	"golang.org/x/net/http2"
@@ -53,14 +53,13 @@ func runWithDispatcher(ctx context.Context) error {
 	opts := []grpc.DialOption{
 		grpc.WithInsecure(),
 	}
-	v1.RegisterWatchServiceServer()
 	// // dial to gRPC for handling request
 	if err := v1.RegisterWatchServiceHandlerFromEndpoint(ctx, gwmux, ":8080", opts); err != nil {
 		return fmt.Errorf("failed registering grpc-gateway: %v", err)
 	}
 	// Register GRPC API
 	gmux := grpc.NewServer()
-	v1.RegisterWatchServiceServer(gmux, &service.WatchService{})
+	v1.RegisterWatchServiceServer(gmux, &watcher.Watcher{})
 	reflection.Register(gmux)
 	server := &http.Server{
 		Addr:    fmt.Sprintf(":%s", port),

diff --git a/services/watcher/cmd/watcher/main.go b/services/watcher/cmd/watcher/main.go
@@ -11,7 +11,6 @@ import (
 	"github.com/GoogleCloudPlatform/functions-framework-go/funcframework"
 	"github.com/grpc-ecosystem/grpc-gateway/runtime"
 	"github.com/shortedapp/shorted/services/watcher/cmd/gcf"
-	"github.com/shortedapp/shorted/services/watcher/internal/service"
 	"github.com/shortedapp/shorted/services/watcher/pkg/config"
 	"github.com/shortedapp/shorted/services/watcher/pkg/log"
 	v1 "github.com/shortedapp/shorted/shortedapis/pkg/watcher/v1"

diff --git a/services/watcher/go.mod b/services/watcher/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
 	github.com/google/uuid v1.1.1
 	github.com/grpc-ecosystem/grpc-gateway v1.15.0
-	github.com/shortedapp/shorted/shortedapis v0.0.0-20200926073704-5473fb39bf82
+	github.com/shortedapp/shorted/shortedapis v0.0.0-20200927141851-06fded6c3942
 	github.com/stretchr/testify v1.6.1
 	go.opencensus.io v0.22.4
 	go.opentelemetry.io/contrib/instrumentation/net/http/httptrace v0.11.0

diff --git a/services/watcher/go.sum b/services/watcher/go.sum
@@ -294,6 +294,8 @@ github.com/shortedapp/shorted/shortedapis v0.0.0-20200926061715-70a81d5d5f30 h1:
 github.com/shortedapp/shorted/shortedapis v0.0.0-20200926061715-70a81d5d5f30/go.mod h1:Pw4nIX7Jm5NbRKZLPRX4ksMDz5Is8gcCZr4ziiVKDV0=
 github.com/shortedapp/shorted/shortedapis v0.0.0-20200926073704-5473fb39bf82 h1:5EhPM0UlBbTfBTApil5XM0WDWuthd6zEspg+yvME6Is=
 github.com/shortedapp/shorted/shortedapis v0.0.0-20200926073704-5473fb39bf82/go.mod h1:h9OBYQjwQ+u+CKHMbJLmLBDrP8AqmkI/y88l16VHkrA=
+github.com/shortedapp/shorted/shortedapis v0.0.0-20200927141851-06fded6c3942 h1:JJnakB2liJn0ouwTS9uLaSPCIzaeJxPFU950+D62KOU=
+github.com/shortedapp/shorted/shortedapis v0.0.0-20200927141851-06fded6c3942/go.mod h1:h9OBYQjwQ+u+CKHMbJLmLBDrP8AqmkI/y88l16VHkrA=
 github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
 github.com/spf13/afero v1.3.4/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

diff --git a/services/watcher/internal/service/watcher/watcher.go b/services/watcher/internal/service/watcher/watcher.go
@@ -45,11 +45,11 @@ func New(ctx context.Context, cfg *config.Config) (*Watcher, error) {
 	// }
 	// log.Infof(ctx, "loaded source: %v", w.Source)
 
-	store, err := store.New("gs://shorted-dev-aba5688f-watcher-index")
+	s, err := store.New("gs://shorted-dev-aba5688f-watcher-index")
 	if err != nil {
 		return &Watcher{}, fmt.Errorf("failed initialising store: %v", err)
 	}
-	w.store = store
+	w.store = s
 	w.Config = cfg
 	w.Context = ctx
 	return &w, nil

diff --git a/services/watcher/sources/asic/asic.go b/services/watcher/sources/asic/asic.go
@@ -86,10 +86,10 @@ func (*handler) Parse(ctx context.Context, s *v1.Source) (*v1.Index, error) {
 		})
 	})
 	si.SetCount(count)
-	log.Infof(ctx, "%d %s documents pulled from %v", count, s.Format, s.URL)
+	log.Infof(ctx, "%d %s documents pulled from %v", count, s.Format, s.Url)
 	log.Response(ctx, response)
 
-	return si, nil
+	return si.ToIndex(), nil
 }
 
 // GetInfo returns the Info associated with this source implementation.

diff --git a/services/watcher/sources/noop/noop.go b/services/watcher/sources/noop/noop.go
@@ -3,15 +3,15 @@ package noop
 import (
 	"context"
 
-	"github.com/shortedapp/shorted/services/watcher/pkg/index"
 	"github.com/shortedapp/shorted/services/watcher/pkg/source"
 	"github.com/shortedapp/shorted/services/watcher/sources/metadata"
+	v1 "github.com/shortedapp/shorted/shortedapis/pkg/watcher/v1"
 )
 
 type handler struct{}
 
-func (*handler) Parse(ctx context.Context, s *source.Source) (*index.Source, error) {
-	return &index.Source{}, nil
+func (*handler) Parse(ctx context.Context, s *v1.Source) (*v1.Index, error) {
+	return &v1.Index{}, nil
 }
 
 // GetInfo returns the Info associated with this source implementation.

diff --git a/shortedapis/Dockerfile b/shortedapis/Dockerfile
@@ -33,6 +33,9 @@ RUN go install \
     github.com/gogo/protobuf/protoc-gen-gogo \
     github.com/gogo/protobuf/gogoproto
 
+RUN go get github.com/gogo/protobuf/protoc-gen-gofast
+
+
 RUN cat go.mod
 
 CMD ["go","generate"]
diff --git a/shortedapis/build.go b/shortedapis/build.go
@@ -1,4 +1,4 @@
 // Watcher
-//go:generate protoc ./proto/shorted/watcher/v1/watcher.proto -I proto/ -I third_party/ --go_out=plugins=grpc:pkg --validate_out=lang=go:pkg
-//go:generate protoc ./proto/shorted/watcher/v1/watcher_service.proto -I proto/ -I third_party/ --go_out=plugins=grpc:pkg --validate_out=lang=go:pkg --grpc-gateway_out=logtostderr=true:pkg --swagger_out=logtostderr=true:pkg
+//go:generate protoc ./proto/shorted/watcher/v1/watcher.proto -I proto/ -I third_party/ --gofast_out=plugins=grpc:pkg --validate_out=lang=go:pkg
+//go:generate protoc ./proto/shorted/watcher/v1/watcher_service.proto -I proto/ -I third_party/ --gofast_out=plugins=grpc:pkg --validate_out=lang=go:pkg --grpc-gateway_out=logtostderr=true:pkg --swagger_out=logtostderr=true:pkg
 package shortedapis