feat(eventindexer): indexing nft metadata (#17538)

Co-authored-by: Jeffery Walsh <cyberhorsey@gmail.com> Co-authored-by: jeff <113397187+cyberhorsey@users.noreply.github.com> Co-authored-by: cyberhorsey <cyberhorsey@users.noreply.github.com>
taikoxyz · Jul 2, 2024 · d0e25ba · d0e25ba
1 parent d2b00ce
commit d0e25ba
Show file tree

Hide file tree

Showing 27 changed files with 991 additions and 100 deletions.
diff --git a/packages/eventindexer/api/api.go b/packages/eventindexer/api/api.go
@@ -72,6 +72,11 @@ func InitFromConfig(ctx context.Context, api *API, cfg *Config) error {
 		return err
 	}
 
+	nftMetadataRepository, err := repo.NewNFTMetadataRepository(db)
+	if err != nil {
+		return err
+	}
+
 	ethClient, err := ethclient.Dial(cfg.RPCUrl)
 	if err != nil {
 		return err
@@ -80,6 +85,7 @@ func InitFromConfig(ctx context.Context, api *API, cfg *Config) error {
 	srv, err := http.NewServer(http.NewServerOpts{
 		EventRepo:        eventRepository,
 		NFTBalanceRepo:   nftBalanceRepository,
+		NFTMetadataRepo:  nftMetadataRepository,
 		ERC20BalanceRepo: erc20BalanceRepository,
 		ChartRepo:        chartRepository,
 		Echo:             echo.New(),

diff --git a/packages/eventindexer/contracts/erc721/abi.go b/packages/eventindexer/contracts/erc721/abi.go
@@ -0,0 +1,39 @@
+package erc721
+
+var (
+	ABI = `[
+		{
+			"constant":true,
+			"inputs":[
+				{
+					"name":"_tokenId",
+					"type":"uint256"
+				}
+			],
+			"name":"tokenURI",
+			"outputs":[
+				{
+					"name":"",
+					"type":"string"
+				}
+			],
+			"payable":false,
+			"stateMutability":"view",
+			"type":"function"
+		},
+		{
+			"constant": true,
+			"inputs": [],
+			"name": "symbol",
+			"outputs": [
+				{
+					"name": "",
+					"type": "string"
+				}
+			],
+			"payable": false,
+			"stateMutability": "view",
+			"type": "function"
+		}
+	]`
+)
diff --git a/packages/eventindexer/errors.go b/packages/eventindexer/errors.go
@@ -12,6 +12,10 @@ var (
 		"ERR_NO_NFT_BALANCE_REPOSITORY",
 		"NFTBalanceRepository is required",
 	)
+	ErrNoNFTMetadataRepository = errors.Validation.NewWithKeyAndDetail(
+		"ERR_NO_NFT_METADATA_REPOSITORY",
+		"NFTMetadataRepository is required",
+	)
 	ErrNoStatRepository = errors.Validation.NewWithKeyAndDetail(
 		"ERR_NO_STAT_REPOSITORY",
 		"StatRepository is required",
@@ -23,4 +27,5 @@ var (
 	ErrNoCORSOrigins = errors.Validation.NewWithKeyAndDetail("ERR_NO_CORS_ORIGINS", "CORS Origins are required")
 	ErrNoRPCClient   = errors.Validation.NewWithKeyAndDetail("ERR_NO_RPC_CLIENT", "RPCClient is required")
 	ErrInvalidMode   = errors.Validation.NewWithKeyAndDetail("ERR_INVALID_MODE", "Mode not supported")
+	ErrInvalidURL    = errors.Validation.NewWithKeyAndDetail("ERR_INVALID_URL", "The provided URL is invalid or unreachable")
 )
diff --git a/packages/eventindexer/indexer/fetch_nft_metadata.go b/packages/eventindexer/indexer/fetch_nft_metadata.go
@@ -0,0 +1,171 @@
+package indexer
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"math/big"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts/abi"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/pkg/errors"
+	"github.com/taikoxyz/taiko-mono/packages/eventindexer"
+	"github.com/taikoxyz/taiko-mono/packages/eventindexer/contracts/erc1155"
+	"github.com/taikoxyz/taiko-mono/packages/eventindexer/contracts/erc721"
+)
+
+func (i *Indexer) fetchNFTMetadata(
+	ctx context.Context, contractAddress string,
+	tokenID *big.Int,
+	abiJSON string,
+	methodName string,
+	chainID *big.Int) (*eventindexer.NFTMetadata, error) {
+	contractABI, err := abi.JSON(strings.NewReader(abiJSON))
+	if err != nil {
+		return nil, err
+	}
+
+	contractAddressCommon := common.HexToAddress(contractAddress)
+
+	callData, err := contractABI.Pack(methodName, tokenID)
+	if err != nil {
+		return nil, err
+	}
+
+	msg := ethereum.CallMsg{
+		To:   &contractAddressCommon,
+		Data: callData,
+	}
+
+	result, err := i.ethClient.CallContract(ctx, msg, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "i.ethClient.CallContract")
+	}
+
+	var tokenURI string
+
+	err = contractABI.UnpackIntoInterface(&tokenURI, methodName, result)
+	if err != nil {
+		return nil, errors.Wrap(err, "contractABI.UnpackIntoInterface")
+	}
+
+	url, err := resolveMetadataURL(ctx, tokenURI)
+	if err != nil {
+		if errors.Is(err, eventindexer.ErrInvalidURL) {
+			slog.Warn("Invalid metadata URI",
+				"contractAddress", contractAddress,
+				"tokenID", tokenID.Int64(),
+				"chainID", chainID.String())
+
+			return nil, nil
+		}
+
+		return nil, errors.Wrap(err, "resolveMetadataURL")
+	}
+
+	//nolint
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	var metadata eventindexer.NFTMetadata
+
+	err = json.NewDecoder(resp.Body).Decode(&metadata)
+	if err != nil {
+		return nil, err
+	}
+
+	if methodName == "tokenURI" {
+		if err := i.fetchSymbol(ctx, contractABI, &metadata, contractAddressCommon); err != nil {
+			return nil, err
+		}
+	}
+
+	metadata.ContractAddress = contractAddress
+	metadata.TokenID = tokenID.Int64()
+	metadata.ChainID = chainID.Int64()
+
+	return &metadata, nil
+}
+
+func resolveMetadataURL(ctx context.Context, tokenURI string) (string, error) {
+	if strings.HasPrefix(tokenURI, "ipfs://") {
+		ipfsHash := strings.TrimPrefix(tokenURI, "ipfs://")
+		resolvedURL := fmt.Sprintf("https://ipfs.io/ipfs/%s", ipfsHash)
+
+		if isValidURL(ctx, resolvedURL) {
+			return resolvedURL, nil
+		}
+
+		return "", eventindexer.ErrInvalidURL
+	}
+
+	if isValidURL(ctx, tokenURI) {
+		return tokenURI, nil
+	}
+
+	return "", eventindexer.ErrInvalidURL
+}
+
+func isValidURL(ctx context.Context, rawURL string) bool {
+	client := &http.Client{
+		Timeout: 3 * time.Second,
+	}
+
+	resp, err := client.Head(rawURL)
+	if err != nil || resp.StatusCode != http.StatusOK {
+		return false
+	}
+
+	return true
+}
+
+func (i *Indexer) fetchSymbol(ctx context.Context, contractABI abi.ABI, metadata *eventindexer.NFTMetadata, contractAddress common.Address) error {
+	symbolCallData, err := contractABI.Pack("symbol")
+	if err != nil {
+		return errors.Wrap(err, "contractABI.Pack")
+	}
+
+	symbolMsg := ethereum.CallMsg{
+		To:   &contractAddress,
+		Data: symbolCallData,
+	}
+
+	symbolResult, err := i.ethClient.CallContract(ctx, symbolMsg, nil)
+	if err != nil {
+		return errors.Wrap(err, "i.ethClient.CallContract(symbolMsg)")
+	}
+
+	var symbol string
+
+	err = contractABI.UnpackIntoInterface(&symbol, "symbol", symbolResult)
+	if err != nil {
+		return errors.Wrap(err, "contractABI.UnpackIntoInterface")
+	}
+
+	metadata.Symbol = symbol
+
+	return nil
+}
+
+func (i *Indexer) fetchERC721Metadata(ctx context.Context,
+	contractAddress string,
+	tokenID *big.Int,
+	chainID *big.Int) (*eventindexer.NFTMetadata, error) {
+	return i.fetchNFTMetadata(ctx, contractAddress, tokenID, erc721.ABI, "tokenURI", chainID)
+}
+
+func (i *Indexer) fetchERC1155Metadata(ctx context.Context,
+	contractAddress string,
+	tokenID *big.Int,
+	chainID *big.Int) (*eventindexer.NFTMetadata, error) {
+	return i.fetchNFTMetadata(ctx, contractAddress, tokenID, erc1155.ABI, "uri", chainID)
+}
diff --git a/packages/eventindexer/indexer/index_erc20_transfers.go b/packages/eventindexer/indexer/index_erc20_transfers.go
@@ -15,6 +15,7 @@ import (
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/pkg/errors"
 	"github.com/taikoxyz/taiko-mono/packages/eventindexer"
+	"golang.org/x/sync/errgroup"
 )
 
 // nolint: lll
@@ -30,14 +31,26 @@ func (i *Indexer) indexERC20Transfers(
 	chainID *big.Int,
 	logs []types.Log,
 ) error {
+	wg, ctx := errgroup.WithContext(ctx)
+
 	for _, vLog := range logs {
-		if !i.isERC20Transfer(ctx, vLog) {
-			continue
-		}
+		l := vLog
 
-		if err := i.saveERC20Transfer(ctx, chainID, vLog); err != nil {
-			return err
-		}
+		wg.Go(func() error {
+			if !i.isERC20Transfer(ctx, l) {
+				return nil
+			}
+
+			if err := i.saveERC20Transfer(ctx, chainID, l); err != nil {
+				return err
+			}
+
+			return nil
+		})
+	}
+
+	if err := wg.Wait(); err != nil {
+		return err
 	}
 
 	return nil
@@ -100,13 +113,27 @@ func (i *Indexer) saveERC20Transfer(ctx context.Context, chainID *big.Int, vLog
 
 	var pk int = 0
 
-	md, err := i.erc20BalanceRepo.FindMetadata(ctx, chainID.Int64(), vLog.Address.Hex())
-	if err != nil {
-		return errors.Wrap(err, "i.erc20BalanceRepo")
+	i.contractToMetadataMutex.Lock()
+
+	md, ok := i.contractToMetadata[vLog.Address]
+
+	i.contractToMetadataMutex.Unlock()
+
+	if !ok {
+		md, err = i.erc20BalanceRepo.FindMetadata(ctx, chainID.Int64(), vLog.Address.Hex())
+		if err != nil {
+			return errors.Wrap(err, "i.erc20BalanceRepo")
+		}
 	}
 
 	if md != nil {
 		pk = md.ID
+
+		i.contractToMetadataMutex.Lock()
+
+		i.contractToMetadata[vLog.Address] = md
+
+		i.contractToMetadataMutex.Unlock()
 	}
 
 	if pk == 0 {