[feature] Custom emoji updates (serve emoji via s2s api, tune db mode…

…ls) (#805)

* migrate emojis

* add get emoji to s2s (federation) API

* add new emoji db + cache functions

* add shortcodeDomain lookup for emojis

* check existing emojis w/cache, not w/constraints

* go fmt

* add putEmoji func

* use new db emoji funcs instead of where

* remove emojistringstotags func

* add unique constraint back in

* fix up broken migration

* update index

internal/api/client/admin/emojicreate_test.go

@@ -29,8 +29,6 @@ import (
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -79,8 +77,7 @@ func (suite *EmojiCreateTestSuite) TestEmojiCreate() {
 	suite.True(apiEmoji.VisibleInPicker)
 
 	// emoji should be in the db
-	dbEmoji := &gtsmodel.Emoji{}
-	err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "shortcode", Value: "new_emoji"}}, dbEmoji)
+	dbEmoji, err := suite.db.GetEmojiByShortcodeDomain(context.Background(), apiEmoji.Shortcode, "")
 	suite.NoError(err)
 
 	// check fields on the emoji

internal/api/s2s/emoji/emoji.go

@@ -0,0 +1,53 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package emoji
+
+import (
+	"net/http"
+
+	"github.com/superseriousbusiness/gotosocial/internal/api"
+	"github.com/superseriousbusiness/gotosocial/internal/processing"
+	"github.com/superseriousbusiness/gotosocial/internal/router"
+	"github.com/superseriousbusiness/gotosocial/internal/uris"
+)
+
+const (
+	// EmojiIDKey is for emoji IDs
+	EmojiIDKey = "id"
+	// EmojiBasePath is the base path for serving information about Emojis eg https://example.org/emoji
+	EmojiWithIDPath = "/" + uris.EmojiPath + "/:" + EmojiIDKey
+)
+
+// Module implements the FederationModule interface
+type Module struct {
+	processor processing.Processor
+}
+
+// New returns a emoji module
+func New(processor processing.Processor) api.FederationModule {
+	return &Module{
+		processor: processor,
+	}
+}
+
+// Route satisfies the RESTAPIModule interface
+func (m *Module) Route(s router.Router) error {
+	s.AttachHandler(http.MethodGet, EmojiWithIDPath, m.EmojiGetHandler)
+	return nil
+}

internal/api/s2s/emoji/emojiget.go

@@ -0,0 +1,74 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package emoji
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/api"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
+)
+
+// EmojiGetHandler
+func (m *Module) EmojiGetHandler(c *gin.Context) {
+	// usernames on our instance are always lowercase
+	requestedEmojiID := strings.ToUpper(c.Param(EmojiIDKey))
+	if requestedEmojiID == "" {
+		err := errors.New("no emoji id specified in request")
+		api.ErrorHandler(c, gtserror.NewErrorBadRequest(err, err.Error()), m.processor.InstanceGet)
+		return
+	}
+
+	format, err := api.NegotiateAccept(c, api.ActivityPubAcceptHeaders...)
+	if err != nil {
+		api.ErrorHandler(c, gtserror.NewErrorNotAcceptable(err, err.Error()), m.processor.InstanceGet)
+		return
+	}
+
+	ctx := c.Request.Context()
+	verifier, signed := c.Get(string(ap.ContextRequestingPublicKeyVerifier))
+	if signed {
+		ctx = context.WithValue(ctx, ap.ContextRequestingPublicKeyVerifier, verifier)
+	}
+
+	signature, signed := c.Get(string(ap.ContextRequestingPublicKeySignature))
+	if signed {
+		ctx = context.WithValue(ctx, ap.ContextRequestingPublicKeySignature, signature)
+	}
+
+	resp, errWithCode := m.processor.GetFediEmoji(ctx, requestedEmojiID, c.Request.URL)
+	if errWithCode != nil {
+		api.ErrorHandler(c, errWithCode, m.processor.InstanceGet)
+		return
+	}
+
+	b, err := json.Marshal(resp)
+	if err != nil {
+		api.ErrorHandler(c, gtserror.NewErrorInternalError(err), m.processor.InstanceGet)
+		return
+	}
+
+	c.Data(http.StatusOK, format, b)
+}

internal/api/s2s/emoji/emojiget_test.go

@@ -0,0 +1,136 @@
+/*
+   GoToSocial
+   Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package emoji_test
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/emoji"
+	"github.com/superseriousbusiness/gotosocial/internal/api/security"
+	"github.com/superseriousbusiness/gotosocial/internal/concurrency"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/email"
+	"github.com/superseriousbusiness/gotosocial/internal/federation"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/messages"
+	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/processing"
+	"github.com/superseriousbusiness/gotosocial/internal/storage"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type EmojiGetTestSuite struct {
+	suite.Suite
+	db             db.DB
+	tc             typeutils.TypeConverter
+	mediaManager   media.Manager
+	federator      federation.Federator
+	emailSender    email.Sender
+	processor      processing.Processor
+	storage        storage.Driver
+	oauthServer    oauth.Server
+	securityModule *security.Module
+
+	testEmojis   map[string]*gtsmodel.Emoji
+	testAccounts map[string]*gtsmodel.Account
+
+	emojiModule *emoji.Module
+}
+
+func (suite *EmojiGetTestSuite) SetupSuite() {
+	suite.testAccounts = testrig.NewTestAccounts()
+	suite.testEmojis = testrig.NewTestEmojis()
+}
+
+func (suite *EmojiGetTestSuite) SetupTest() {
+	testrig.InitTestConfig()
+	testrig.InitTestLog()
+
+	clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
+	fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
+
+	suite.db = testrig.NewTestDB()
+	suite.tc = testrig.NewTestTypeConverter(suite.db)
+	suite.storage = testrig.NewInMemoryStorage()
+	suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
+	suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
+	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
+	suite.emojiModule = emoji.New(suite.processor).(*emoji.Module)
+	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
+	suite.securityModule = security.New(suite.db, suite.oauthServer).(*security.Module)
+	testrig.StandardDBSetup(suite.db, suite.testAccounts)
+	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
+}
+
+func (suite *EmojiGetTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+	testrig.StandardStorageTeardown(suite.storage)
+}
+
+func (suite *EmojiGetTestSuite) TestGetEmoji() {
+	// the dereference we're gonna use
+	derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts)
+	signedRequest := derefRequests["foss_satan_dereference_emoji"]
+	targetEmoji := suite.testEmojis["rainbow"]
+
+	// setup request
+	recorder := httptest.NewRecorder()
+	ctx, _ := testrig.CreateGinTestContext(recorder, nil)
+	ctx.Request = httptest.NewRequest(http.MethodGet, targetEmoji.URI, nil) // the endpoint we're hitting
+	ctx.Request.Header.Set("accept", "application/activity+json")
+	ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
+	ctx.Request.Header.Set("Date", signedRequest.DateHeader)
+
+	// we need to pass the context through signature check first to set appropriate values on it
+	suite.securityModule.SignatureCheck(ctx)
+
+	// normally the router would populate these params from the path values,
+	// but because we're calling the function directly, we need to set them manually.
+	ctx.Params = gin.Params{
+		gin.Param{
+			Key:   emoji.EmojiIDKey,
+			Value: targetEmoji.ID,
+		},
+	}
+
+	// trigger the function being tested
+	suite.emojiModule.EmojiGetHandler(ctx)
+
+	// check response
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+	b, err := ioutil.ReadAll(result.Body)
+	suite.NoError(err)
+
+	suite.Contains(string(b), `"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji"`)
+}
+
+func TestEmojiGetTestSuite(t *testing.T) {
+	suite.Run(t, new(EmojiGetTestSuite))
+}