Implement RemoveDocument API with IsRemoved mark in docInfo on PushPull

.yorkie/config.json

@@ -0,0 +1 @@
+{"auths":{"localhost:11103":"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2NzgxNzg1ODYsInVzZXJuYW1lIjoiYWRtaW4ifQ.oTUzmoIczpw6il7NSkMHc2wiqxfazM1mqd1eOPrsfz0"}}

api/types/auth_webhook.go

@@ -54,6 +54,7 @@ const (
 	DeactivateClient Method = "DeactivateClient"
 	AttachDocument   Method = "AttachDocument"
 	DetachDocument   Method = "DetachDocument"
+	RemoveDocument   Method = "RemoveDocument"
 	PushPull         Method = "PushPull"
 	WatchDocuments   Method = "WatchDocuments"
 )
@@ -75,6 +76,7 @@ func AuthMethods() []Method {
 		DeactivateClient,
 		AttachDocument,
 		DetachDocument,
+		RemoveDocument,
 		PushPull,
 		WatchDocuments,
 	}

server/backend/database/client_info.go

@@ -78,7 +78,7 @@ type ClientInfo struct {
 }
 
 // CheckIfInProject checks if the client is in the project.
-func (i ClientInfo) CheckIfInProject(projectID types.ID) error {
+func (i *ClientInfo) CheckIfInProject(projectID types.ID) error {
 	if i.ProjectID != projectID {
 		return ErrClientNotFound
 	}

server/backend/database/database.go

@@ -168,6 +168,7 @@ type Database interface {
 		docInfo *DocInfo,
 		initialServerSeq int64,
 		changes []*change.Change,
+		removeDoc bool,
 	) error
 
 	// PurgeStaleChanges delete changes before the smallest in `syncedseqs` to

server/backend/database/doc_info.go

@@ -40,6 +40,9 @@ type DocInfo struct {
 	// Owner is the owner(ID of the client) of the document.
 	Owner types.ID `bson:"owner"`
 
+	// IsRemoved represents whether document is removed.
+	IsRemoved bool `bson:"is_removed"`
+
 	// CreatedAt is the time when the document is created.
 	CreatedAt time.Time `bson:"created_at"`
 
@@ -68,6 +71,7 @@ func (info *DocInfo) DeepCopy() *DocInfo {
 		Key:        info.Key,
 		ServerSeq:  info.ServerSeq,
 		Owner:      info.Owner,
+		IsRemoved:  info.IsRemoved,
 		CreatedAt:  info.CreatedAt,
 		AccessedAt: info.AccessedAt,
 		UpdatedAt:  info.UpdatedAt,

server/backend/database/memory/database.go

@@ -631,7 +631,7 @@ func (d *DB) FindDocInfoByKeyAndOwner(
 	txn := d.db.Txn(true)
 	defer txn.Abort()
 
-	raw, err := txn.First(tblDocuments, "project_id_key", projectID.String(), key.String())
+	raw, err := txn.First(tblDocuments, "project_id_key_is_removed", projectID.String(), key.String(), false)
 	if err != nil {
 		return nil, fmt.Errorf("find document by key: %w", err)
 	}
@@ -671,7 +671,7 @@ func (d *DB) FindDocInfoByKey(
 	txn := d.db.Txn(false)
 	defer txn.Abort()
 
-	raw, err := txn.First(tblDocuments, "project_id_key", projectID.String(), key.String())
+	raw, err := txn.First(tblDocuments, "project_id_key_is_removed", projectID.String(), key.String(), false)
 	if err != nil {
 		return nil, fmt.Errorf("find document by key: %w", err)
 	}
@@ -703,13 +703,15 @@ func (d *DB) FindDocInfoByID(
 	return docInfo.DeepCopy(), nil
 }
 
-// CreateChangeInfos stores the given changes and doc info.
+// CreateChangeInfos stores the given changes and doc info. If the
+// removeDoc condition is true, mark IsRemoved to true in doc info.
 func (d *DB) CreateChangeInfos(
 	ctx context.Context,
 	projectID types.ID,
 	docInfo *database.DocInfo,
 	initialServerSeq int64,
 	changes []*change.Change,
+	removeDoc bool,
 ) error {
 	txn := d.db.Txn(true)
 	defer txn.Abort()
@@ -734,7 +736,7 @@ func (d *DB) CreateChangeInfos(
 		}
 	}
 
-	raw, err := txn.First(tblDocuments, "project_id_key", projectID.String(), docInfo.Key.String())
+	raw, err := txn.First(tblDocuments, "project_id_key_is_removed", projectID.String(), docInfo.Key.String(), false)
 	if err != nil {
 		return fmt.Errorf("find document by key: %w", err)
 	}
@@ -747,6 +749,7 @@ func (d *DB) CreateChangeInfos(
 	}
 
 	loadedDocInfo.ServerSeq = docInfo.ServerSeq
+	loadedDocInfo.IsRemoved = removeDoc
 	loadedDocInfo.UpdatedAt = gotime.Now()
 	if err := txn.Insert(tblDocuments, loadedDocInfo); err != nil {
 		return fmt.Errorf("update document: %w", err)

server/backend/database/memory/database_test.go

@@ -178,6 +178,53 @@ func TestDB(t *testing.T) {
 		assert.Equal(t, 15, res.TotalCount)
 	})
 
+	t.Run("mark IsRemoved in docInfo test", func(t *testing.T) {
+		docKey := key.Key(fmt.Sprintf("tests$%s", t.Name()))
+
+		clientInfo, _ := db.ActivateClient(ctx, projectID, t.Name())
+		docInfo, _ := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo.ID, docKey, true)
+		assert.NoError(t, clientInfo.AttachDocument(docInfo.ID))
+		assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo))
+
+		doc := document.New(key.Key(t.Name()))
+		pack := doc.CreateChangePack()
+
+		// Mark docInfo as removed and store changes
+		err = db.CreateChangeInfos(ctx, projectID, docInfo, 0, pack.Changes, true)
+		assert.NoError(t, err)
+
+		// Check whether docInfo is marked as removed
+		docInfo, err = db.FindDocInfoByID(ctx, docInfo.ID)
+		assert.NoError(t, err)
+		assert.Equal(t, true, docInfo.IsRemoved)
+	})
+
+	t.Run("reuse same key to create docInfo test ", func(t *testing.T) {
+		docKey := key.Key(fmt.Sprintf("tests$%s", t.Name()))
+
+		clientInfo1, _ := db.ActivateClient(ctx, projectID, t.Name())
+		docInfo1, _ := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo1.ID, docKey, true)
+		assert.NoError(t, clientInfo1.AttachDocument(docInfo1.ID))
+		assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo1, docInfo1))
+
+		doc := document.New(key.Key(t.Name()))
+		pack := doc.CreateChangePack()
+
+		// Mark docInfo as removed and store changes
+		err = db.CreateChangeInfos(ctx, projectID, docInfo1, 0, pack.Changes, true)
+		assert.NoError(t, err)
+
+		// Use same key to create docInfo
+		clientInfo2, _ := db.ActivateClient(ctx, projectID, t.Name())
+		docInfo2, _ := db.FindDocInfoByKeyAndOwner(ctx, projectID, clientInfo2.ID, docKey, true)
+		assert.NoError(t, clientInfo2.AttachDocument(docInfo2.ID))
+		assert.NoError(t, db.UpdateClientInfoAfterPushPull(ctx, clientInfo2, docInfo2))
+
+		// Check they have same key but different id
+		assert.Equal(t, docInfo1.Key, docInfo2.Key)
+		assert.NotEqual(t, docInfo1.ID, docInfo2.ID)
+	})
+
 	t.Run("update clientInfo after PushPull test", func(t *testing.T) {
 		clientInfo, err := db.ActivateClient(ctx, projectID, t.Name())
 		assert.NoError(t, err)
@@ -220,7 +267,7 @@ func TestDB(t *testing.T) {
 		}
 
 		// Store changes
-		err = db.CreateChangeInfos(ctx, projectID, docInfo, 0, pack.Changes)
+		err = db.CreateChangeInfos(ctx, projectID, docInfo, 0, pack.Changes, false)
 		assert.NoError(t, err)
 
 		// Find changes

server/backend/database/memory/indexes.go

@@ -124,12 +124,21 @@ var schema = &memdb.DBSchema{
 					},
 				},
 				"project_id_key": {
-					Name:   "project_id_key",
-					Unique: true,
+					Name: "project_id_key",
+					Indexer: &memdb.CompoundIndex{
+						Indexes: []memdb.Indexer{
+							&memdb.StringFieldIndex{Field: "ProjectID"},
+							&memdb.StringFieldIndex{Field: "Key"},
+						},
+					},
+				},
+				"project_id_key_is_removed": {
+					Name: "project_id_key_is_removed",
 					Indexer: &memdb.CompoundIndex{
 						Indexes: []memdb.Indexer{
 							&memdb.StringFieldIndex{Field: "ProjectID"},
 							&memdb.StringFieldIndex{Field: "Key"},
+							&memdb.BoolFieldIndex{Field: "IsRemoved"},
 						},
 					},
 				},

server/backend/database/mongo/client.go

@@ -698,6 +698,7 @@ func (c *Client) FindDocInfoByKeyAndOwner(
 	res, err := c.collection(colDocuments).UpdateOne(ctx, bson.M{
 		"project_id": encodedProjectID,
 		"key":        docKey,
+		"is_removed": false,
 	}, bson.M{
 		"$set": bson.M{
 			"accessed_at": now,
@@ -723,6 +724,7 @@ func (c *Client) FindDocInfoByKeyAndOwner(
 		result = c.collection(colDocuments).FindOne(ctx, bson.M{
 			"project_id": encodedProjectID,
 			"key":        docKey,
+			"is_removed": false,
 		})
 		if result.Err() == mongo.ErrNoDocuments {
 			return nil, fmt.Errorf("%s %s: %w", projectID, docKey, database.ErrDocumentNotFound)
@@ -755,6 +757,7 @@ func (c *Client) FindDocInfoByKey(
 	result := c.collection(colDocuments).FindOne(ctx, bson.M{
 		"project_id": encodedProjectID,
 		"key":        docKey,
+		"is_removed": false,
 	})
 	if result.Err() == mongo.ErrNoDocuments {
 		return nil, fmt.Errorf("%s %s: %w", projectID, docKey, database.ErrDocumentNotFound)
@@ -806,40 +809,43 @@ func (c *Client) CreateChangeInfos(
 	docInfo *database.DocInfo,
 	initialServerSeq int64,
 	changes []*change.Change,
+	removeDoc bool,
 ) error {
 	encodedDocID, err := encodeID(docInfo.ID)
 	if err != nil {
 		return err
 	}
 
-	var models []mongo.WriteModel
-	for _, cn := range changes {
-		encodedOperations, err := database.EncodeOperations(cn.Operations())
-		if err != nil {
-			return err
+	if !removeDoc || len(changes) > 0 {
+		var models []mongo.WriteModel
+		for _, cn := range changes {
+			encodedOperations, err := database.EncodeOperations(cn.Operations())
+			if err != nil {
+				return err
+			}
+
+			models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{
+				"doc_id":     encodedDocID,
+				"server_seq": cn.ServerSeq(),
+			}).SetUpdate(bson.M{"$set": bson.M{
+				"actor_id":   encodeActorID(cn.ID().ActorID()),
+				"client_seq": cn.ID().ClientSeq(),
+				"lamport":    cn.ID().Lamport(),
+				"message":    cn.Message(),
+				"operations": encodedOperations,
+			}}).SetUpsert(true))
 		}
 
-		models = append(models, mongo.NewUpdateOneModel().SetFilter(bson.M{
-			"doc_id":     encodedDocID,
-			"server_seq": cn.ServerSeq(),
-		}).SetUpdate(bson.M{"$set": bson.M{
-			"actor_id":   encodeActorID(cn.ID().ActorID()),
-			"client_seq": cn.ID().ClientSeq(),
-			"lamport":    cn.ID().Lamport(),
-			"message":    cn.Message(),
-			"operations": encodedOperations,
-		}}).SetUpsert(true))
-	}
-
-	// TODO(hackerwins): We need to handle the updates for the two collections
-	// below atomically.
-	if _, err = c.collection(colChanges).BulkWrite(
-		ctx,
-		models,
-		options.BulkWrite().SetOrdered(true),
-	); err != nil {
-		logging.From(ctx).Error(err)
-		return fmt.Errorf("bulk write changes: %w", err)
+		// TODO(hackerwins): We need to handle the updates for the two collections
+		// below atomically.
+		if _, err = c.collection(colChanges).BulkWrite(
+			ctx,
+			models,
+			options.BulkWrite().SetOrdered(true),
+		); err != nil {
+			logging.From(ctx).Error(err)
+			return fmt.Errorf("bulk write changes: %w", err)
+		}
 	}
 
 	res, err := c.collection(colDocuments).UpdateOne(ctx, bson.M{
@@ -848,6 +854,7 @@ func (c *Client) CreateChangeInfos(
 	}, bson.M{
 		"$set": bson.M{
 			"server_seq": docInfo.ServerSeq,
+			"is_removed": removeDoc,
 			"updated_at": gotime.Now(),
 		},
 	})

server/backend/database/mongo/client_test.go

@@ -18,8 +18,12 @@ package mongo_test
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
+	"github.com/yorkie-team/yorkie/pkg/document"
+	"github.com/yorkie-team/yorkie/pkg/document/key"
+
 	"github.com/stretchr/testify/assert"
 
 	"github.com/yorkie-team/yorkie/api/types"
@@ -41,6 +45,7 @@ func TestClient(t *testing.T) {
 	cli, err := mongo.Dial(config)
 	assert.NoError(t, err)
 
+	dummyProjectID := types.ID("000000000000000000000000")
 	dummyOwnerID := types.ID("000000000000000000000000")
 	otherOwnerID := types.ID("000000000000000000000001")
 	clientDeactivateThreshold := "1h"
@@ -119,4 +124,52 @@ func TestClient(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Equal(t, info1.ID, info2.ID)
 	})
+
+	t.Run("mark IsRemoved in docInfo test", func(t *testing.T) {
+		docKey := key.Key(fmt.Sprintf("tests$%s", t.Name()))
+
+		clientInfo, _ := cli.ActivateClient(ctx, dummyProjectID, t.Name())
+		docInfo, _ := cli.FindDocInfoByKeyAndOwner(ctx, dummyProjectID, clientInfo.ID, docKey, true)
+		assert.NoError(t, clientInfo.AttachDocument(docInfo.ID))
+		assert.NoError(t, cli.UpdateClientInfoAfterPushPull(ctx, clientInfo, docInfo))
+
+		doc := document.New(key.Key(t.Name()))
+		pack := doc.CreateChangePack()
+
+		// Mark docInfo as removed and store changes
+		err = cli.CreateChangeInfos(ctx, dummyProjectID, docInfo, 0, pack.Changes, true)
+		assert.NoError(t, err)
+
+		// Check whether docInfo is marked as removed
+		docInfo, err = cli.FindDocInfoByID(ctx, docInfo.ID)
+		assert.NoError(t, err)
+		assert.Equal(t, true, docInfo.IsRemoved)
+	})
+
+	t.Run("reuse same key to create docInfo test ", func(t *testing.T) {
+		docKey := key.Key(fmt.Sprintf("tests$%s", t.Name()))
+
+		clientInfo1, _ := cli.ActivateClient(ctx, dummyProjectID, t.Name())
+		docInfo1, _ := cli.FindDocInfoByKeyAndOwner(ctx, dummyProjectID, clientInfo1.ID, docKey, true)
+		assert.NoError(t, clientInfo1.AttachDocument(docInfo1.ID))
+		assert.NoError(t, cli.UpdateClientInfoAfterPushPull(ctx, clientInfo1, docInfo1))
+
+		doc := document.New(key.Key(t.Name()))
+		pack := doc.CreateChangePack()
+
+		// Mark docInfo as removed and store changes
+		// This will not work because there are no changes in pack
+		err = cli.CreateChangeInfos(ctx, dummyProjectID, docInfo1, 0, pack.Changes, true)
+		assert.NoError(t, err)
+
+		// Use same key to create docInfo
+		clientInfo2, _ := cli.ActivateClient(ctx, dummyProjectID, t.Name())
+		docInfo2, _ := cli.FindDocInfoByKeyAndOwner(ctx, dummyProjectID, clientInfo2.ID, docKey, true)
+		assert.NoError(t, clientInfo2.AttachDocument(docInfo2.ID))
+		assert.NoError(t, cli.UpdateClientInfoAfterPushPull(ctx, clientInfo2, docInfo2))
+
+		// Check they have same key but different id
+		assert.Equal(t, docInfo1.Key, docInfo2.Key)
+		assert.NotEqual(t, docInfo1.ID, docInfo2.ID)
+	})
 }

server/backend/database/mongo/indexes.go

@@ -87,7 +87,12 @@ var collectionInfos = []collectionInfo{
 				{Key: "project_id", Value: bsonx.Int32(1)},
 				{Key: "key", Value: bsonx.Int32(1)},
 			},
-			Options: options.Index().SetUnique(true),
+		}, {
+			Keys: bsonx.Doc{
+				{Key: "project_id", Value: bsonx.Int32(1)},
+				{Key: "key", Value: bsonx.Int32(1)},
+				{Key: "is_removed", Value: bsonx.Int32(1)},
+			},
 		}},
 	}, {
 		name: colChanges,