feat: Public key store/cache for Observer

Implemented a public key storage and cache for the Observer. When verifying a witness proof, the Observer will load the public key of the signer from the cache/database. If it doesn't exist in the DB then the public key is retrieved using the public key fetcher.

Also enhanced the BDD test so that, when creating DIDs on multiple targets, the target will be greylisted if it is down and the create will be performed on another target.

closes #1261

Signed-off-by: Bob Stasyszyn <Bob.Stasyszyn@securekey.com>

cmd/orb-server/startcmd/start.go

@@ -18,6 +18,8 @@ import (
 	"encoding/pem"
 	"errors"
 	"fmt"
+	"github.com/hyperledger/aries-framework-go/pkg/doc/signature/verifier"
+	"github.com/trustbloc/orb/pkg/store/publickey"
 	"net"
 	"net/http"
 	"net/url"
@@ -870,6 +872,15 @@ func startOrbServices(parameters *orbParameters) error {
 		return fmt.Errorf("open store: %w", err)
 	}
 
+	pkStore, err := publickey.New(storeProviders.provider, verifiable.NewVDRKeyResolver(vdr).PublicKeyFetcher())
+	if err != nil {
+		return fmt.Errorf("create public key storage: %w", err)
+	}
+
+	publicKeyFetcher := func(issuerID, keyID string) (*verifier.PublicKey, error) {
+		return pkStore.GetPublicKey(issuerID, keyID)
+	}
+
 	// create new observer and start it
 	providers := &observer.Providers{
 		ProtocolClientProvider: pcp,
@@ -881,7 +892,7 @@ func startOrbServices(parameters *orbParameters) error {
 		WebFingerResolver:      resourceResolver,
 		CASResolver:            casResolver,
 		DocLoader:              orbDocumentLoader,
-		Pkf:                    verifiable.NewVDRKeyResolver(vdr).PublicKeyFetcher(),
+		PublicKeyFetcher:       publicKeyFetcher,
 		AnchorLinkStore:        anchorLinkStore,
 	}
 
@@ -994,6 +1005,7 @@ func startOrbServices(parameters *orbParameters) error {
 		discoveryclient.WithNamespace(parameters.didNamespace),
 		discoveryclient.WithHTTPClient(httpClient),
 		discoveryclient.WithDIDWebHTTP(parameters.enableDevMode),
+		discoveryclient.WithPublicKeyFetcher(publicKeyFetcher),
 	)
 
 	if parameters.verifyLatestFromAnchorOrigin {

pkg/discovery/endpoint/client/client.go

@@ -68,6 +68,7 @@ type Client struct {
 	casReader         casReader
 	authToken         string
 	disableProofCheck bool
+	publicKeyFetcher  verifiable.PublicKeyFetcher
 	didWebHTTP        bool
 	docLoader         ld.DocumentLoader
 	orbClient         orbClient
@@ -108,13 +109,16 @@ func New(docLoader ld.DocumentLoader, casReader casReader, opts ...Option) (*Cli
 	if configService.disableProofCheck {
 		orbClientOpts = append(orbClientOpts, aoprovider.WithDisableProofCheck(configService.disableProofCheck))
 	} else {
-		orbClientOpts = append(orbClientOpts, aoprovider.WithPublicKeyFetcher(
-			verifiable.NewVDRKeyResolver(vdr.New(vdr.WithVDR(&webVDR{
+		if configService.publicKeyFetcher == nil {
+			configService.publicKeyFetcher = verifiable.NewVDRKeyResolver(vdr.New(vdr.WithVDR(&webVDR{
 				http:    configService.httpClient,
 				useHTTP: configService.didWebHTTP,
 				VDR:     web.New(),
 			}),
-			)).PublicKeyFetcher()))
+			)).PublicKeyFetcher()
+		}
+
+		orbClientOpts = append(orbClientOpts, aoprovider.WithPublicKeyFetcher(configService.publicKeyFetcher))
 	}
 
 	orbClient, err := aoprovider.New(configService.namespace, configService.casReader, orbClientOpts...)
@@ -498,6 +502,14 @@ func WithDisableProofCheck(disable bool) Option {
 	}
 }
 
+// WithPublicKeyFetcher sets the public key fetcher. If not set then
+// the default fetcher is used.
+func WithPublicKeyFetcher(pkf verifiable.PublicKeyFetcher) Option {
+	return func(opts *Client) {
+		opts.publicKeyFetcher = pkf
+	}
+}
+
 // WithDIDWebHTTP use did web http.
 func WithDIDWebHTTP(enable bool) Option {
 	return func(opts *Client) {

pkg/discovery/endpoint/client/client_test.go

@@ -16,6 +16,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hyperledger/aries-framework-go/pkg/doc/signature/verifier"
 	"github.com/stretchr/testify/require"
 
 	"github.com/trustbloc/orb/pkg/activitypub/client/transport"
@@ -48,6 +49,20 @@ func TestNew(t *testing.T) {
 		require.Equal(t, time.Minute, cs.cacheLifetime)
 		require.Equal(t, 500, cs.cacheSize)
 	})
+
+	t.Run("success - with public key fetcher", func(t *testing.T) {
+		cs, err := New(nil, &referenceCASReaderImplementation{},
+			WithAuthToken("t1"),
+			WithPublicKeyFetcher(func(issuerID, keyID string) (*verifier.PublicKey, error) {
+				return &verifier.PublicKey{}, nil
+			}),
+		)
+		require.NoError(t, err)
+		require.NotNil(t, cs)
+
+		require.Equal(t, defaultCacheLifetime, cs.cacheLifetime)
+		require.Equal(t, defaultCacheSize, cs.cacheSize)
+	})
 }
 
 func TestConfigService_GetEndpointAnchorOrigin(t *testing.T) {

pkg/observer/observer.go

@@ -29,7 +29,7 @@ import (
 	"github.com/trustbloc/orb/pkg/anchor/subject"
 	"github.com/trustbloc/orb/pkg/anchor/util"
 	discoveryrest "github.com/trustbloc/orb/pkg/discovery/endpoint/restapi"
-	"github.com/trustbloc/orb/pkg/errors"
+	orberrors "github.com/trustbloc/orb/pkg/errors"
 	"github.com/trustbloc/orb/pkg/hashlink"
 	"github.com/trustbloc/orb/pkg/linkset"
 	"github.com/trustbloc/orb/pkg/pubsub/spi"
@@ -135,7 +135,7 @@ type Providers struct {
 	WebFingerResolver resourceResolver
 	CASResolver       casResolver
 	DocLoader         documentLoader
-	Pkf               verifiable.PublicKeyFetcher
+	PublicKeyFetcher  verifiable.PublicKeyFetcher
 	AnchorLinkStore   anchorLinkStore
 }
 
@@ -253,7 +253,7 @@ func (o *Observer) processDID(did string) error {
 		if err := o.processAnchor(
 			&anchorinfo.AnchorInfo{Hashlink: anchor.CID},
 			anchor.Info, suffix); err != nil {
-			if errors.IsTransient(err) {
+			if orberrors.IsTransient(err) {
 				// Return an error so that the message is redelivered and retried.
 				return fmt.Errorf("process out-of-system anchor [%s]: %w", anchor.CID, err)
 			}
@@ -313,7 +313,7 @@ func (o *Observer) processAnchor(anchor *anchorinfo.AnchorInfo,
 	}
 
 	vc, err := util.VerifiableCredentialFromAnchorLink(anchorLink,
-		verifiable.WithPublicKeyFetcher(o.Pkf),
+		verifiable.WithPublicKeyFetcher(o.PublicKeyFetcher),
 		verifiable.WithJSONLDDocumentLoader(o.DocLoader),
 	)
 	if err != nil {