ocis crashed: fatal error: concurrent map read and map write

[saw-jan]

Build: https://drone.owncloud.com/owncloud/ocis/28565/30/3

fatal error: concurrent map read and map write

goroutine 93618 [running]:
github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache.(*Cache).List(0xc00077a130, {0x4455cf0, 0xc004d8fe60}, {0xc0015ed920, 0x24})
	/drone/src/vendor/github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go:195 +0x325
github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3.(*Manager).ListReceivedShares(0xc00077a000, {0x4455cf0, 0xc004d8fe30}, {0xc000c977f8, 0x1, 0x1}, 0xc004ee4010?)
	/drone/src/vendor/github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/jsoncs3.go:806 +0x446
github.com/cs3org/reva/v2/internal/grpc/services/usershareprovider.(*service).ListReceivedShares(0xc0024ac2a0, {0x4455cf0, 0xc004d8fe30}, 0xc0041a4cc0)
	/drone/src/vendor/github.com/cs3org/reva/v2/internal/grpc/services/usershareprovider/usershareprovider.go:252 +0x19b
github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1._CollaborationAPI_ListReceivedShares_Handler.func1({0x4455cf0, 0xc004d8fe30}, {0x30a4b60?, 0xc0041a4cc0})
	/drone/src/vendor/github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1/collaboration_api.pb.go:1501 +0x78
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x4455cf0, 0xc004d8fd70}, {0x30a4b60, 0xc0041a4cc0}, 0xc003ffeae0, 0xc003945188)
	/drone/src/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:341 +0x437
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func2.1({0x4455cf0?, 0xc004d8fd70?}, {0x30a4b60?, 0xc0041a4cc0?})
	/drone/src/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:48 +0x4f
github.com/cs3org/reva/v2/internal/grpc/interceptors/eventsmiddleware.NewUnary.func1({0x4455cf0, 0xc004d8fd40}, {0x30a4b60?, 0xc0041a4cc0?}, 0x30?, 0xc003f4cc80)
	/drone/src/vendor/github.com/cs3org/reva/v2/internal/grpc/interceptors/eventsmiddleware/events.go:66 +0xe8
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func2.1({0x4455cf0?, 0xc004d8fd40?}, {0x30a4b60?, 0xc0041a4cc0?})
	/drone/src/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:48 +0x4f
github.com/cs3org/reva/v2/internal/grpc/interceptors/prometheus.interceptorFromConfig.func1({0x4455cf0?, 0xc004d8fd40?}, {0x30a4b60?, 0xc0041a4cc0?}, 0x2bbcb40?, 0x4426801?)

Another: https://drone.owncloud.com/owncloud/ocis/28454/33/4

fatal error: concurrent map read and map write

goroutine 257168 [running]:
github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache.(*Cache).persist(0xc001c12130, {0x443deb0, 0xc002fc3f50}, {0xc006f55350, 0x24})
	/drone/src/vendor/github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go:287 +0x50e
github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache.(*Cache).Add.func1()
	/drone/src/vendor/github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go:124 +0x32b
github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache.(*Cache).Add(0xc001c12130, {0x443deb0, 0xc0065676b0}, {0xc006f55350, 0x24}, {0xc006c89d10, 0x49}, 0xc006056340)
	/drone/src/vendor/github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go:134 +0xb48
github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3.(*Manager).UpdateReceivedShare(0xc001c12000, {0x443deb0, 0xc006567650}, 0xc0062d9400, 0xc0062d9480, 0xc002da0280)
	/drone/src/vendor/github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/jsoncs3.go:1022 +0x4d5
github.com/cs3org/reva/v2/internal/grpc/services/usershareprovider.(*service).UpdateReceivedShare(0xc001e96f00, {0x443deb0, 0xc006567650}, 0xc0062d93c0)
	/drone/src/vendor/github.com/cs3org/reva/v2/internal/grpc/services/usershareprovider/usershareprovider.go:309 +0x1b6
github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1._CollaborationAPI_UpdateReceivedShare_Handler.func1({0x443deb0, 0xc006567650}, {0x306ed00?, 0xc0062d93c0})
	/drone/src/vendor/github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1/collaboration_api.pb.go:1519 +0x78
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x443deb0, 0xc006567590}, {0x306ed00, 0xc0062d93c0}, 0xc006295360, 0xc004c27d88)

[saw-jan]

CC @micbar @kobergj

[rhafer]

@aduffeck This seems to happen cs3share cache. Any idea?

[2403905]

It cannot reproduce isolated.
The code also looks good.
@saw-jan Please add a full failure-log if you face the race condition next time.
How I tested:
go test -count 1000 -parallel 8 ./... -run "^TestCache_*"

package receivedsharecache_test

import (
	"context"
	"os"
	"testing"
	"time"

	collaboration "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
	"github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache"
	"github.com/cs3org/reva/v2/pkg/storage/utils/metadata"
)

func InitCache() receivedsharecache.Cache {
	tmpdir, _ := os.MkdirTemp("", "providercache-test")
	os.MkdirAll(tmpdir, 0755)
	storage, _ := metadata.NewDiskStorage(tmpdir)
	return receivedsharecache.New(storage, 0*time.Second)
}

var (
	c = InitCache()

	userID  = "user"
	spaceID = "spaceid"
	shareID = "storageid$spaceid!share1"
	share   = &collaboration.Share{
		Id: &collaboration.ShareId{
			OpaqueId: shareID,
		},
	}
	ctx = context.Background()
)

func TestCache_Add(t *testing.T) {
	t.Parallel()
	rs := &collaboration.ReceivedShare{
		Share: share,
		State: collaboration.ShareState_SHARE_STATE_PENDING,
	}
	t.Run("add", func(t *testing.T) {
		if err := c.Add(ctx, userID, spaceID, rs); err != nil {
			t.Errorf("Add() error = %v", err)
		}
	})
}

func TestCache_Get(t *testing.T) {
	t.Parallel()
	t.Run("get", func(t *testing.T) {
		_, err := c.Get(ctx, userID, spaceID, shareID)
		if err != nil {
			t.Errorf("Get() error = %v", err)
			return
		}
	})
}

func TestCache_List(t *testing.T) {
	t.Parallel()
	t.Run("List", func(t *testing.T) {
		_, err := c.List(ctx, userID)
		if err != nil {
			t.Errorf("List() error = %v", err)
			return
		}
	})
}

[saw-jan]

It cannot reproduce isolated.
The code also looks good.
Please add a full failure-log if you face the race condition next time.

The logs in the builds have expired. And also I haven't seen this error for a while.
I will report full log if it happens again. Thanks

[saw-jan]

Similar crash: https://drone.owncloud.com/owncloud/ocis/30164/29/3

fatal error: concurrent map read and map write

goroutine 366773 [running]:
github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache.(*Cache).Add(0xc001e7c670, {0x44826b8, 0xc004f7aae0}, {0xc005262ab0, 0x24}, {0xc002366780, 0x49}, 0xc004025700)
	/drone/src/vendor/github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go:100 +0x2fd
github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3.(*Manager).UpdateReceivedShare(0xc001e7c540, {0x44826b8, 0xc004f7aa80}, 0xc0020939c0, 0xc002093ac0, 0xc0042289b0)
	/drone/src/vendor/github.com/cs3org/reva/v2/pkg/share/manager/jsoncs3/jsoncs3.go:1022 +0x4b7

[kobergj]

Locking is insufficient in the cache. We need to replace https://github.com/cs3org/reva/blob/edge/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go#L49 with a cornelk/hashmap to avoid the concurrent writes

[aduffeck]

Or use our mtimesyncedcache package as in providercache.go.

[saw-jan]

Seen today as well:

• https://drone.owncloud.com/owncloud/ocis/30271/37/3
• https://drone.owncloud.com/owncloud/ocis/30279/36/3
• https://drone.owncloud.com/owncloud/ocis/30285/36/3