[feature] Show + federate emojis in accounts (#837)

* Start adding account emoji

* get emojis serialized + deserialized nicely

* update tests

* set / retrieve emojis on accounts

* show account emojis in web view

* fetch emojis from db based on ids

* fix typo in test

* lint

* fix pg migration

* update tests

* update emoji checking logic

* update comment

* clarify comments + add some spacing

* tidy up loops a lil (thanks kim)

internal/ap/interfaces.go

@@ -41,6 +41,7 @@ type Accountable interface {
 	WithFeatured
 	WithManuallyApprovesFollowers
 	WithEndpoints
+	WithTag
 }
 
 // Statusable represents the minimum activitypub interface for representing a 'status'.

internal/api/client/account/accountupdate_test.go

@@ -200,7 +200,7 @@ func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandlerGet
 func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandlerTwoFields() {
 	// set up the request
 	// we're updating the note of zork, and setting locked to true
-	newBio := "this is my new bio read it and weep"
+	newBio := "this is my new bio read it and weep :rainbow:"
 	requestBody, w, err := testrig.CreateMultipartFormData(
 		"", "",
 		map[string]string{
@@ -235,9 +235,19 @@ func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandlerTwo
 
 	// check the returned api model account
 	// fields should be updated
-	suite.Equal("<p>this is my new bio read it and weep</p>", apimodelAccount.Note)
+	suite.Equal("<p>this is my new bio read it and weep :rainbow:</p>", apimodelAccount.Note)
 	suite.Equal(newBio, apimodelAccount.Source.Note)
 	suite.True(apimodelAccount.Locked)
+	suite.NotEmpty(apimodelAccount.Emojis)
+	suite.Equal(apimodelAccount.Emojis[0].Shortcode, "rainbow")
+
+	// check the account in the database
+	dbZork, err := suite.db.GetAccountByID(context.Background(), apimodelAccount.ID)
+	suite.NoError(err)
+	suite.Equal(newBio, dbZork.NoteRaw)
+	suite.Equal("<p>this is my new bio read it and weep :rainbow:</p>", dbZork.Note)
+	suite.True(*dbZork.Locked)
+	suite.NotEmpty(dbZork.EmojiIDs)
 }
 
 func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandlerWithMedia() {

internal/api/s2s/user/inboxpost_test.go

@@ -237,6 +237,8 @@ func (suite *InboxPostTestSuite) TestPostUnblock() {
 func (suite *InboxPostTestSuite) TestPostUpdate() {
 	updatedAccount := *suite.testAccounts["remote_account_1"]
 	updatedAccount.DisplayName = "updated display name!"
+	testEmoji := testrig.NewTestEmojis()["rainbow"]
+	updatedAccount.Emojis = []*gtsmodel.Emoji{testEmoji}
 
 	asAccount, err := suite.tc.AccountToAS(context.Background(), &updatedAccount)
 	suite.NoError(err)
@@ -288,6 +290,15 @@ func (suite *InboxPostTestSuite) TestPostUpdate() {
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
 	emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
+	if err := processor.Start(); err != nil {
+		panic(err)
+	}
+	defer func() {
+		if err := processor.Stop(); err != nil {
+			panic(err)
+		}
+	}()
+
 	userModule := user.New(processor).(*user.Module)
 
 	// setup request
@@ -322,11 +333,21 @@ func (suite *InboxPostTestSuite) TestPostUpdate() {
 	suite.Equal(http.StatusOK, result.StatusCode)
 
 	// account should be changed in the database now
-	dbUpdatedAccount, err := suite.db.GetAccountByID(context.Background(), updatedAccount.ID)
-	suite.NoError(err)
+	var dbUpdatedAccount *gtsmodel.Account
+
+	if !testrig.WaitFor(func() bool {
+		// displayName should be updated
+		dbUpdatedAccount, _ = suite.db.GetAccountByID(context.Background(), updatedAccount.ID)
+		return dbUpdatedAccount.DisplayName == "updated display name!"
+	}) {
+		suite.FailNow("timed out waiting for account update")
+	}
+
+	// emojis should be updated
+	suite.Contains(dbUpdatedAccount.EmojiIDs, testEmoji.ID)
 
-	// displayName should be updated
-	suite.Equal("updated display name!", dbUpdatedAccount.DisplayName)
+	// account should be freshly webfingered
+	suite.WithinDuration(time.Now(), dbUpdatedAccount.LastWebfingeredAt, 10*time.Second)
 
 	// everything else should be the same as it was before
 	suite.EqualValues(updatedAccount.Username, dbUpdatedAccount.Username)
@@ -350,7 +371,6 @@ func (suite *InboxPostTestSuite) TestPostUpdate() {
 	suite.EqualValues(updatedAccount.Language, dbUpdatedAccount.Language)
 	suite.EqualValues(updatedAccount.URI, dbUpdatedAccount.URI)
 	suite.EqualValues(updatedAccount.URL, dbUpdatedAccount.URL)
-	suite.EqualValues(updatedAccount.LastWebfingeredAt, dbUpdatedAccount.LastWebfingeredAt)
 	suite.EqualValues(updatedAccount.InboxURI, dbUpdatedAccount.InboxURI)
 	suite.EqualValues(updatedAccount.OutboxURI, dbUpdatedAccount.OutboxURI)
 	suite.EqualValues(updatedAccount.FollowingURI, dbUpdatedAccount.FollowingURI)

internal/cache/account.go

@@ -116,6 +116,8 @@ func copyAccount(account *gtsmodel.Account) *gtsmodel.Account {
 		HeaderMediaAttachment:   nil,
 		HeaderRemoteURL:         account.HeaderRemoteURL,
 		DisplayName:             account.DisplayName,
+		EmojiIDs:                account.EmojiIDs,
+		Emojis:                  nil,
 		Fields:                  account.Fields,
 		Note:                    account.Note,
 		NoteRaw:                 account.NoteRaw,

internal/db/account.go

@@ -42,6 +42,9 @@ type Account interface {
 	// GetAccountByPubkeyID returns one account with the given public key URI (ID), or an error if something goes wrong.
 	GetAccountByPubkeyID(ctx context.Context, id string) (*gtsmodel.Account, Error)
 
+	// PutAccount puts one account in the database.
+	PutAccount(ctx context.Context, account *gtsmodel.Account) (*gtsmodel.Account, Error)
+
 	// UpdateAccount updates one account by ID.
 	UpdateAccount(ctx context.Context, account *gtsmodel.Account) (*gtsmodel.Account, Error)
 

internal/db/bundb/account.go

@@ -45,7 +45,8 @@ func (a *accountDB) newAccountQ(account *gtsmodel.Account) *bun.SelectQuery {
 		NewSelect().
 		Model(account).
 		Relation("AvatarMediaAttachment").
-		Relation("HeaderMediaAttachment")
+		Relation("HeaderMediaAttachment").
+		Relation("Emojis")
 }
 
 func (a *accountDB) GetAccountByID(ctx context.Context, id string) (*gtsmodel.Account, db.Error) {
@@ -138,24 +139,61 @@ func (a *accountDB) getAccount(ctx context.Context, cacheGet func() (*gtsmodel.A
 	return account, nil
 }
 
+func (a *accountDB) PutAccount(ctx context.Context, account *gtsmodel.Account) (*gtsmodel.Account, db.Error) {
+	if err := a.conn.RunInTx(ctx, func(tx bun.Tx) error {
+		// create links between this account and any emojis it uses
+		for _, i := range account.EmojiIDs {
+			if _, err := tx.NewInsert().Model(&gtsmodel.AccountToEmoji{
+				AccountID: account.ID,
+				EmojiID:   i,
+			}).Exec(ctx); err != nil {
+				return err
+			}
+		}
+
+		// insert the account
+		_, err := tx.NewInsert().Model(account).Exec(ctx)
+		return err
+	}); err != nil {
+		return nil, a.conn.ProcessError(err)
+	}
+
+	a.cache.Put(account)
+	return account, nil
+}
+
 func (a *accountDB) UpdateAccount(ctx context.Context, account *gtsmodel.Account) (*gtsmodel.Account, db.Error) {
 	// Update the account's last-updated
 	account.UpdatedAt = time.Now()
 
-	// Update the account model in the DB
-	_, err := a.conn.
-		NewUpdate().
-		Model(account).
-		WherePK().
-		Exec(ctx)
-	if err != nil {
+	if err := a.conn.RunInTx(ctx, func(tx bun.Tx) error {
+		// create links between this account and any emojis it uses
+		// first clear out any old emoji links
+		if _, err := tx.NewDelete().
+			Model(&[]*gtsmodel.AccountToEmoji{}).
+			Where("account_id = ?", account.ID).
+			Exec(ctx); err != nil {
+			return err
+		}
+
+		// now populate new emoji links
+		for _, i := range account.EmojiIDs {
+			if _, err := tx.NewInsert().Model(&gtsmodel.AccountToEmoji{
+				AccountID: account.ID,
+				EmojiID:   i,
+			}).Exec(ctx); err != nil {
+				return err
+			}
+		}
+
+		// update the account
+		_, err := tx.NewUpdate().Model(account).WherePK().Exec(ctx)
+		return err
+	}); err != nil {
 		return nil, a.conn.ProcessError(err)
 	}
 
-	// Place updated account in cache
-	// (this will replace existing, i.e. invalidating)
 	a.cache.Put(account)
-
 	return account, nil
 }
 

internal/db/bundb/account_test.go

@@ -27,7 +27,9 @@ import (
 
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/db/bundb"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/uptrace/bun"
 )
 
 type AccountTestSuite struct {
@@ -71,17 +73,70 @@ func (suite *AccountTestSuite) TestGetAccountByUsernameDomain() {
 }
 
 func (suite *AccountTestSuite) TestUpdateAccount() {
+	ctx := context.Background()
+
 	testAccount := suite.testAccounts["local_account_1"]
 
 	testAccount.DisplayName = "new display name!"
+	testAccount.EmojiIDs = []string{"01GD36ZKWTKY3T1JJ24JR7KY1Q", "01GD36ZV904SHBHNAYV6DX5QEF"}
+
+	_, err := suite.db.UpdateAccount(ctx, testAccount)
+	suite.NoError(err)
+
+	updated, err := suite.db.GetAccountByID(ctx, testAccount.ID)
+	suite.NoError(err)
+	suite.Equal("new display name!", updated.DisplayName)
+	suite.Equal([]string{"01GD36ZKWTKY3T1JJ24JR7KY1Q", "01GD36ZV904SHBHNAYV6DX5QEF"}, updated.EmojiIDs)
+	suite.WithinDuration(time.Now(), updated.UpdatedAt, 5*time.Second)
+
+	// get account without cache + make sure it's really in the db as desired
+	dbService, ok := suite.db.(*bundb.DBService)
+	if !ok {
+		panic("db was not *bundb.DBService")
+	}
+
+	noCache := &gtsmodel.Account{}
+	err = dbService.GetConn().
+		NewSelect().
+		Model(noCache).
+		Where("account.id = ?", bun.Ident(testAccount.ID)).
+		Relation("AvatarMediaAttachment").
+		Relation("HeaderMediaAttachment").
+		Relation("Emojis").
+		Scan(ctx)
+
+	suite.NoError(err)
+	suite.Equal("new display name!", noCache.DisplayName)
+	suite.Equal([]string{"01GD36ZKWTKY3T1JJ24JR7KY1Q", "01GD36ZV904SHBHNAYV6DX5QEF"}, noCache.EmojiIDs)
+	suite.WithinDuration(time.Now(), noCache.UpdatedAt, 5*time.Second)
+	suite.NotNil(noCache.AvatarMediaAttachment)
+	suite.NotNil(noCache.HeaderMediaAttachment)
 
-	_, err := suite.db.UpdateAccount(context.Background(), testAccount)
+	// update again to remove emoji associations
+	testAccount.EmojiIDs = []string{}
+
+	_, err = suite.db.UpdateAccount(ctx, testAccount)
 	suite.NoError(err)
 
-	updated, err := suite.db.GetAccountByID(context.Background(), testAccount.ID)
+	updated, err = suite.db.GetAccountByID(ctx, testAccount.ID)
 	suite.NoError(err)
 	suite.Equal("new display name!", updated.DisplayName)
+	suite.Empty(updated.EmojiIDs)
 	suite.WithinDuration(time.Now(), updated.UpdatedAt, 5*time.Second)
+
+	err = dbService.GetConn().
+		NewSelect().
+		Model(noCache).
+		Where("account.id = ?", bun.Ident(testAccount.ID)).
+		Relation("AvatarMediaAttachment").
+		Relation("HeaderMediaAttachment").
+		Relation("Emojis").
+		Scan(ctx)
+
+	suite.NoError(err)
+	suite.Equal("new display name!", noCache.DisplayName)
+	suite.Empty(noCache.EmojiIDs)
+	suite.WithinDuration(time.Now(), noCache.UpdatedAt, 5*time.Second)
 }
 
 func (suite *AccountTestSuite) TestInsertAccountWithDefaults() {

internal/db/bundb/bundb.go

@@ -67,12 +67,13 @@ const (
 )
 
 var registerTables = []interface{}{
+	&gtsmodel.AccountToEmoji{},
 	&gtsmodel.StatusToEmoji{},
 	&gtsmodel.StatusToTag{},
 }
 
-// bunDBService satisfies the DB interface
-type bunDBService struct {
+// DBService satisfies the DB interface
+type DBService struct {
 	db.Account
 	db.Admin
 	db.Basic
@@ -89,6 +90,12 @@ type bunDBService struct {
 	conn *DBConn
 }
 
+// GetConn returns the underlying bun connection.
+// Should only be used in testing + exceptional circumstance.
+func (dbService *DBService) GetConn() *DBConn {
+	return dbService.conn
+}
+
 func doMigration(ctx context.Context, db *bun.DB) error {
 	migrator := migrate.NewMigrator(db, migrations.Migrations)
 
@@ -177,7 +184,7 @@ func NewBunDBService(ctx context.Context) (db.DB, error) {
 	// Prepare domain block cache
 	blockCache := cache.NewDomainBlockCache()
 
-	ps := &bunDBService{
+	ps := &DBService{
 		Account: accounts,
 		Admin: &adminDB{
 			conn: conn,
@@ -399,7 +406,7 @@ func tweakConnectionValues(sqldb *sql.DB) {
 	CONVERSION FUNCTIONS
 */
 
-func (ps *bunDBService) TagStringsToTags(ctx context.Context, tags []string, originAccountID string) ([]*gtsmodel.Tag, error) {
+func (dbService *DBService) TagStringsToTags(ctx context.Context, tags []string, originAccountID string) ([]*gtsmodel.Tag, error) {
 	protocol := config.GetProtocol()
 	host := config.GetHost()
 
@@ -408,7 +415,7 @@ func (ps *bunDBService) TagStringsToTags(ctx context.Context, tags []string, ori
 		tag := &gtsmodel.Tag{}
 		// we can use selectorinsert here to create the new tag if it doesn't exist already
 		// inserted will be true if this is a new tag we just created
-		if err := ps.conn.NewSelect().Model(tag).Where("LOWER(?) = LOWER(?)", bun.Ident("name"), t).Scan(ctx); err != nil {
+		if err := dbService.conn.NewSelect().Model(tag).Where("LOWER(?) = LOWER(?)", bun.Ident("name"), t).Scan(ctx); err != nil {
 			if err == sql.ErrNoRows {
 				// tag doesn't exist yet so populate it
 				newID, err := id.NewRandomULID()