diff --git a/apps/web/src/composables/contracts.ts b/apps/web/src/composables/contracts.ts index 7ad11ef12..6d68da9dd 100644 --- a/apps/web/src/composables/contracts.ts +++ b/apps/web/src/composables/contracts.ts @@ -48,23 +48,29 @@ export default function useContracts() { const { isWalletConnectSigner, getEthersWalletConnectSigner } = useWalletConnect() async function deposit({ amount, walletProvider }: { amount: string, walletProvider: ProviderString }) { - // const ethAmount = (parseInt(amount) / (await getCurrentPrice({ coin: 'ETH', currency: 'USD' }))).toString() - const signerCreators = { - 'Browser': getEthersBrowserSigner, - 'Ledger': getEthersLedgerSigner, - 'Trezor': getEthersTrezorSigner, - 'WalletConnect': getEthersWalletConnectSigner + try { + // const ethAmount = (parseInt(amount) / (await getCurrentPrice({ coin: 'ETH', currency: 'USD' }))).toString() + const signerCreators = { + 'Browser': getEthersBrowserSigner, + 'Ledger': getEthersLedgerSigner, + 'Trezor': getEthersTrezorSigner, + 'WalletConnect': getEthersWalletConnectSigner + } + const signerType = ethersProviderList.includes(walletProvider) ? 'Browser' : walletProvider + const signerCreator = signerCreators[signerType as keyof typeof signerCreators] + let signer = signerCreator(walletProvider) + if (isWalletConnectSigner(signer)) signer = await signer + const managerSigner = manager.connect(signer as ethers.Signer) + const fees = await managerSigner.feePercent() + const depositAmount = parseFloat(amount) * ((100 + fees) / 100) + const value = ethers.utils.parseEther(depositAmount.toString()) + const result = await managerSigner.depositStake({ value, type: 0 }) + await result.wait() + return true + } catch (err) { + console.error(`There was an error in despoit function: ${err}`) + return false } - const signerType = ethersProviderList.includes(walletProvider) ? 'Browser' : walletProvider - const signerCreator = signerCreators[signerType as keyof typeof signerCreators] - let signer = signerCreator(walletProvider) - if (isWalletConnectSigner(signer)) signer = await signer - const managerSigner = manager.connect(signer as ethers.Signer) - const fees = await managerSigner.feePercent() - const depositAmount = parseFloat(amount) * ((100 + fees) / 100) - const value = ethers.utils.parseEther(depositAmount.toString()) - const result = await managerSigner.depositStake({ value, type: 0 }) - return await result.wait() } async function getCurrentStaked() : Promise { diff --git a/apps/web/src/pages/overview/components/Staking.vue b/apps/web/src/pages/overview/components/Staking.vue index 912013628..f70e84ae2 100644 --- a/apps/web/src/pages/overview/components/Staking.vue +++ b/apps/web/src/pages/overview/components/Staking.vue @@ -2,20 +2,24 @@ import { ref, onMounted, onUnmounted, watch } from 'vue' import { FormattedWalletOption, ProviderString } from '@casimir/types' import VueFeather from 'vue-feather' +import usePrice from '@/composables/price' import useEthers from '@/composables/ethers' import useUsers from '@/composables/users' import useContracts from '@/composables/contracts' import TermsOfService from '@/components/TermsOfService.vue' +const { deposit, getDepositFees } = useContracts() const { getEthersBalance } = useEthers() const { user, getUserAnalytics } = useUsers() -const { deposit, withdraw } = useContracts() +const { getCurrentPrice } = usePrice() const selectedProvider = ref('') const selectedWallet = ref(null as null | string) const formattedAmountToStake = ref('') const address_balance = ref(null as null | string) +const currentEthPrice = ref(0) +const estimatedFees = ref('-') const openSelectWalletInput = ref(false) @@ -101,7 +105,6 @@ const aggregateAddressesByProvider = () => { } watch(selectedWallet, async () => { - // const currentEthPrice = await getCurrentPrice({coin: 'ETH', currency: 'USD'}) address_balance.value = selectedWallet.value ? (Math.round( await getEthersBalance(selectedWallet.value) * 100) / 100 ) + ' ETH': '- - -' }) @@ -124,7 +127,7 @@ watch(formattedAmountToStake, async () => { }else { errorMessage.value = null } - }else{ + } else{ errorMessage.value = null } }) @@ -133,11 +136,14 @@ watch(user, () => { aggregateAddressesByProvider() }) -onMounted(() => { +onMounted(async () => { window.addEventListener('click', handleOutsideClick) aggregateAddressesByProvider() + currentEthPrice.value = Math.round((await getCurrentPrice({coin: 'ETH', currency: 'USD'})) * 100) / 100 + estimatedFees.value = await getDepositFees() }) + onUnmounted(() =>{ window.removeEventListener('click', handleOutsideClick) }) @@ -146,37 +152,31 @@ const loading = ref(false) const success = ref(false) const failure = ref(false) const stakeButtonText = ref('Stake') -const handleDeposit = () => { - deposit({ amount: formattedAmountToStake.value, walletProvider: selectedProvider.value }) - - const isSuccess = Math.random() < 0.5 // Replace with your actual logic +const handleDeposit = async () => { loading.value = true + const isSuccess = await deposit({ amount: formattedAmountToStake.value, walletProvider: selectedProvider.value }) + loading.value = false + if (isSuccess) { + success.value = true + stakeButtonText.value = 'Transaction Successfully Submitted' + } else { + failure.value = true + stakeButtonText.value = 'Transaction Failed' + } setTimeout(() => { - - loading.value = false - if (isSuccess) { - success.value = true - stakeButtonText.value = 'Success' - } else { - failure.value = true - stakeButtonText.value = 'Transaction Failed' - } - - setTimeout(() => { - success.value = false - failure.value = false - stakeButtonText.value = 'Stake' - - // empty out staking comp - selectedProvider.value = '' - selectedWallet.value = null - formattedAmountToStake.value = '' - address_balance.value = null - - }, 3000) - }, 2000) + success.value = false + failure.value = false + stakeButtonText.value = 'Stake' + + // empty out staking comp + selectedProvider.value = '' + selectedWallet.value = null + formattedAmountToStake.value = '' + address_balance.value = null + + }, 3000) @@ -305,17 +305,17 @@ const handleDeposit = () => {
- 0.0002 ETH + {{ estimatedFees }}.00%
- Exchange Price + Exchange Rate
- 1 USD - 0.000ETH + ${{ currentEthPrice }}/ETH
@@ -362,13 +362,6 @@ const handleDeposit = () => { {{ stakeButtonText }}
- -
bs.End { - return fmt.Errorf("start must be less than end") + err = crawler.Crawl() + + if err != nil { + return err } - return nil + defer crawler.Close() -} + // streamer, err := NewEthereumStreamer() + + // if err != nil { + // return err + // } -func (bs *BaseConfig) String() string { - return fmt.Sprintf("chain: %s network: %s url: %s \n", bs.Chain, bs.Network, bs.Url) + // for now use crawler's introspect and s3 client rather than recreating them + // streamer.Glue = crawler.Glue + // streamer.S3 = crawler.S3 + + // err = streamer.Stream() + + // if err != nil { + // return err + // } + + return nil } diff --git a/services/crawler/client.go b/services/crawler/client.go deleted file mode 100644 index cdc2dacc0..000000000 --- a/services/crawler/client.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/athena" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/ethereum/go-ethereum/ethclient" - "net/http" - "os" - "time" -) - -func PString(s string) *string { - return &s -} - -func NewEthereumClient(url string) (*ethclient.Client, error) { - client, err := ethclient.Dial(url) - if err != nil { - return nil, err - } - return client, nil -} - -func NewS3Client() (*s3.S3, error) { - sess, err := session.NewSession(&aws.Config{ - Region: PString(os.Getenv("AWS_REGION")), - Credentials: credentials.NewEnvCredentials(), - }, - ) - if err != nil { - return nil, err - } - return s3.New(sess), nil -} - -func NewAthenaClient() (*athena.Athena, error) { - sess, err := session.NewSession(&aws.Config{ - Region: PString(os.Getenv("AWS_REGION")), - Credentials: credentials.NewEnvCredentials(), - }, - ) - if err != nil { - return nil, err - } - - return athena.New(sess), nil -} - -func NewHTTPClient() (*http.Client, error) { - client := &http.Client{ - Timeout: time.Second * 10, - } - return client, nil -} diff --git a/services/crawler/crawler.go b/services/crawler/crawler.go index 707db9dc6..844da52aa 100644 --- a/services/crawler/crawler.go +++ b/services/crawler/crawler.go @@ -6,518 +6,579 @@ import ( "encoding/json" "errors" "fmt" - "io" "math/big" - "net/http" + "net/url" "os" + "strconv" + "strings" "sync" "time" - "github.com/xitongsys/parquet-go/parquet" - "github.com/xitongsys/parquet-go/writer" + "github.com/ethereum/go-ethereum/core/types" + "github.com/schollz/progressbar" +) + +type TxDirection string +type OperationType string - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/ethereum/go-ethereum/ethclient" - _ "github.com/segmentio/go-athena" +const ( + Outgoing TxDirection = "outgoing" + Incoming TxDirection = "incoming" + ConcurrencyLimit = 200 + AWSAthenaTimeFormat = "2006-01-02 15:04:05.999999999" ) -type EthereumCrawler struct { - BaseConfig - EthClient *ethclient.Client - S3Client *s3.S3 - Elapsed time.Duration - TotalBlocks int64 - ParquetWriter *writer.ParquetWriter - HttpClient *http.Client +type Chain struct { + Name ChainType + Network NetworkType } -type ChainErr struct { - Block int64 `json:"block"` - Error error `json:"error"` +type Event struct { + Chain ChainType `json:"chain"` + Network NetworkType `json:"network"` + Provider ProviderType `json:"provider"` + Type EventType `json:"type"` + Height int64 `json:"height"` + Block string `json:"block"` + Transaction string `json:"transaction"` + ReceivedAt string `json:"received_at"` + Sender string `json:"sender" ` + Recipient string `json:"recipient"` + Amount string `json:"amount"` + Price float64 `json:"price"` + SenderBalance string `json:"sender_balance"` + RecipientBalance string `json:"recipient_balance"` + GasFee string `json:"gas_fee"` } -func NewEthereumCrawler(config BaseConfig) (*EthereumCrawler, error) { - if config.Chain == Ethereum { - ethClient, err := NewEthereumClient(config.Url) - - if err != nil { - panic(err) - } - - if err != nil { - return nil, err - } +type WalletEvent struct { + WalletAddress string `json:"wallet_address"` + Balance string `json:"wallet_balance"` + Direction TxDirection `json:"tx_direction"` + TxId string `json:"tx_id"` + ReceivedAt string `json:"received_at"` + Amount string `json:"amount"` + Price float64 `json:"price"` + GasFee string `json:"gas_fee"` +} - s3Client, err := NewS3Client() +type StakingActionEvent struct { + WalletAddress string `json:"wallet_address"` + StakeDeposit int64 `json:"stake_deposit"` + CreatedAt string `json:"created_at"` + StakeRebalance int64 `json:"stake_rebalance"` + WithdrawalAmount int64 `json:"withdrawal_amount"` + DistributeReward int64 `json:"distribute_reward"` +} +type PkgJSON struct { + Version string `json:"version"` +} - if err != nil { - return nil, err - } +type Table struct { + Name string + Database string + Version string + Bucket string + SerDe string +} - crawler := EthereumCrawler{ - BaseConfig: config, - EthClient: ethClient, - S3Client: s3Client, - } +type Crawler interface { + Crawl() error + Close() error +} - http, err := NewHTTPClient() +type EthereumCrawler struct { + EthereumClient + Logger + Mutex *sync.Mutex + Begin time.Time + Elapsed time.Duration + Glue *GlueClient + S3 *S3Client + Exchange Exchange + Sema chan struct{} + Wg *sync.WaitGroup + Head uint64 + EventsConsumed int + Version int + Progress *progressbar.ProgressBar + // block and tx events + txBucket string + walletBucket string + stakingBucket string +} - if err != nil { - return nil, err - } +func NewEthereumCrawler() (*EthereumCrawler, error) { + raw := os.Getenv("ETHEREUM_RPC") - crawler.HttpClient = http - - schema := []*parquet.SchemaElement{ - { - Name: "chain", - Type: parquet.TypePtr(parquet.Type_BYTE_ARRAY), - }, - { - Name: "network", - Type: parquet.TypePtr(parquet.Type_BYTE_ARRAY), - }, - { - Name: "provider", - Type: parquet.TypePtr(parquet.Type_BYTE_ARRAY), - }, - { - Name: "type", - Type: parquet.TypePtr(parquet.Type_BYTE_ARRAY), - }, - { - Name: "height", - Type: parquet.TypePtr(parquet.Type_INT64), - }, - { - Name: "block", - Type: parquet.TypePtr(parquet.Type_BYTE_ARRAY), - }, - { - Name: "transaction", - Type: parquet.TypePtr(parquet.Type_BYTE_ARRAY), - }, - { - Name: "created_at", - Type: parquet.TypePtr(parquet.Type_BYTE_ARRAY), - }, - { - Name: "address", - Type: parquet.TypePtr(parquet.Type_BYTE_ARRAY), - }, - { - Name: "to_address", - Type: parquet.TypePtr(parquet.Type_BYTE_ARRAY), - }, - { - Name: "amount", - Type: parquet.TypePtr(parquet.Type_BYTE_ARRAY), - }, - { - Name: "address_balance", - Type: parquet.TypePtr(parquet.Type_BYTE_ARRAY), - }, - { - Name: "to_address_balance", - Type: parquet.TypePtr(parquet.Type_BYTE_ARRAY), - }, - { - Name: "price", - Type: parquet.TypePtr(parquet.Type_DOUBLE), - }, - } + if raw == "" { + return nil, errors.New("ETHEREUM_RPC env variable is not set") + } - pw, err := NewParquetWriter(schema) + url, err := url.Parse(raw) - if err != nil { - return nil, err - } - - crawler.ParquetWriter = pw + if err != nil { + return nil, err + } - block, err := crawler.EthClient.HeaderByNumber(context.Background(), nil) + client, err := NewEthereumClient(Casimir, *url) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } - crawler.Head = block.Number.Int64() + head, err := client.Client.BlockNumber(context.Background()) - return &crawler, nil + if err != nil { + return nil, err } - return nil, errors.New("unsupported chain") -} -func (e *EthereumCrawler) Crawl() error { - e.Print("process id: %d \n", os.Getpid()) - s3Files, err := e.ListS3Files() + config, err := LoadDefaultAWSConfig() if err != nil { - return err + return nil, err } - intervals := sliceRange(14000000, 14000000+1000, 100) + glue, err := NewGlueClient(config) - if len(s3Files) > 0 { - fmt.Println("some files already exists in s3") + if err != nil { + return nil, err } - var wg sync.WaitGroup - - var errs []ChainErr - errChan := make(chan ChainErr) + s3c, err := NewS3Client() - e.Print("crawling ethereum blocks...\n") + if err != nil { + return nil, err + } - begin := time.Now() + key := os.Getenv("CRYPTOCOMPARE_API_KEY") - for _, v := range intervals { - wg.Add(1) - go e.Fetch(&wg, errChan, v) + if key == "" { + return nil, errors.New("CRYPTOCOMPARE_API_KEY env variable is not set") } - go func(ch chan ChainErr) { - for { - select { - case err := <-ch: - fmt.Printf("error: block %d, %v \n", err.Block, err.Error.Error()) - errs = append(errs, ChainErr{ - Block: err.Block, - Error: err.Error, - }) - return - } - } - }(errChan) - - defer func() { - if len(errs) > 0 { - errorsByte, err := json.Marshal(errs) + exchange, err := NewCryptoCompareExchange(key) - errorsByte = append(errorsByte, []byte(" \n")...) + if err != nil { + return nil, err + } - if err != nil { - panic(err) - } + _, err = exchange.CurrentPrice(Ethereum, USD) - err = SaveErrorLog(errorsByte) + if err != nil { + return nil, err + } - if err != nil { - panic(err) - } - e.Print("errors logged to %s.log \n", "crawler") - } - elapsed := time.Since(begin) - e.Elapsed = elapsed - e.Print("elapsed time: %v \n", elapsed) - }() - wg.Wait() - return nil + return &EthereumCrawler{ + EthereumClient: *client, + Logger: NewStdoutLogger(), + Mutex: &sync.Mutex{}, + Sema: make(chan struct{}, ConcurrencyLimit), + Glue: glue, + S3: s3c, + Head: head, + Wg: &sync.WaitGroup{}, + Begin: time.Now(), + }, nil } -func (e *EthereumCrawler) Fetch(wg *sync.WaitGroup, errChan chan ChainErr, blockRange []int64) { - e.Print("new thread crawling blocks: %d - %d\n", blockRange[0], blockRange[1]) - - defer wg.Done() +func (c *EthereumCrawler) Crawl() error { + l := c.Logger - bachSize := 500 - start := blockRange[0] - end := blockRange[1] + err := c.Introspect() - var events []Event + if err != nil { + return nil + } - for i := start; i <= end; i++ { - //fmt.Printf("crawling block: %d\n", i) - block, err := e.EthClient.BlockByNumber(context.Background(), big.NewInt(i)) + l.Info("crawling %d blocks...\n", c.Head+1) - if err != nil { - errChan <- ChainErr{ - Block: i, - Error: err, - } - } + step := 250_000 - blockEvent, err := NewBlockEvent(block) + for i := int(c.Head); i >= 0; i -= step { + start := i + end := i - step + 1 - if err != nil { - errChan <- ChainErr{ - Block: i, - Error: err, - } + if end < 0 { + end = 0 } - //price, err := e.CurrentPrice("USD", Ethereum) + c.Wg.Add(1) + go func(start, end int) { + defer func() { + c.Wg.Done() + <-c.Sema + l.Info("completed batch=%d start=%d end=%d\n", i/step, start, end) + }() - //if err != nil { - // errChan <- ChainErr{ - // Block: i, - // Error: err, - // } - //} - // - //blockEvent.Price = price.Value - - events = append(events, blockEvent) - - if len(block.Transactions()) > 0 { - for k, tx := range block.Transactions() { - _, err := e.EthClient.TransactionReceipt(context.Background(), tx.Hash()) + l.Info("started batch=%d start=%d end=%d\n", i/step, start, end) + for j := start; j >= end; j-- { + txEvents, walletEvents, err := c.ProcessBlock(int(j)) if err != nil { - fmt.Println("cant get tx receipt") - errChan <- ChainErr{ - Block: i, - Error: err, - } + l.Error("block=%d error=%s\n", j, err.Error()) + continue } - txEvent := NewTxEvent(block, tx) + if len(txEvents) != 0 { + err = c.SaveTxEvents(int(j), txEvents) - if err != nil { - fmt.Println("cant get tx event") - errChan <- ChainErr{ - Block: i, - Error: err, + if err != nil { + l.Error("block=%d error=%s\n", j, err.Error()) + continue } } - //txEvent.Price = price.Value - - if tx.To() != nil { - balance, err := e.EthClient.BalanceAt(context.Background(), *tx.To(), nil) + if len(walletEvents) != 0 { + err = c.SaveWalletEvents(int(j), walletEvents) if err != nil { - fmt.Println("cant get balance for to address") - errChan <- ChainErr{ - Block: i, - Error: err, - } + l.Error("block=%d error=%s\n", j, err.Error()) + continue } - txEvent.Recipient = tx.To().Hex() - txEvent.RecipientBalance = balance.Int64() + } + // c.Mutex.Lock() + // c.EventsConsumed += len(txEvents) + len(walletEvents) + // c.Mutex.Unlock() + } + }(start, end) + } + return nil +} - //txIndex := k - len(block.Transactions()) - sender, err := e.EthClient.TransactionSender(context.Background(), tx, block.Hash(), uint(k)) +func (c *EthereumCrawler) SaveTxEvents(block int, tx []*Event) error { + var txEvents bytes.Buffer - if err != nil { - fmt.Println("cant get sender") - errChan <- ChainErr{ - Block: i, - Error: err, - } - } + if len(tx) == 0 { + return errors.New("events are empty") + } - balance, err = e.EthClient.BalanceAt(context.Background(), sender, block.Number()) + for _, e := range tx { + b, err := json.Marshal(e) - if err != nil { - fmt.Println("cant get balance for sender") - errChan <- ChainErr{ - Block: i, - Error: err, - } - } - txEvent.Sender = sender.Hex() - txEvent.SenderBalance = balance.Int64() - } - events = append(events, txEvent) - } + if err != nil { + return err } - //if i%int64(bachSize) == 0 { - if i%int64(bachSize) == 0 && i != start { - eventsND, err := NDJSON(&events) - if err != nil { - panic(err) - } + txEvents.Write(b) + txEvents.WriteByte('\n') - key := fmt.Sprintf("%d-%d.ndjson", blockRange[0], i) + } - data := []byte(eventsND) + if c.txBucket[len(c.txBucket)-1] == '/' { + c.txBucket = c.txBucket[:len(c.txBucket)-1] + } - err = e.SaveToS3(&key, &data) + dest := fmt.Sprintf("%s/%s/block=%d.ndjson", Ethereum.String(), c.Network.String(), block) - if err != nil { - errChan <- ChainErr{ - Block: i, - Error: err, - } - } + err := c.S3.UploadBytes(c.txBucket, dest, &txEvents) - e.Print("saved %s \n", key) - } + if err != nil { + return err } - if len(events) > 0 { - eventsND, err := NDJSON(&events) + c.Logger.Info("uploaded %d tx events to %s\n", len(tx), dest) + return nil +} - if err != nil { - panic(err) - } +func (c *EthereumCrawler) SaveWalletEvents(block int, wallet []*WalletEvent) error { + if len(wallet) == 0 { + return errors.New("events are empty") + } - key := fmt.Sprintf("%d-%d.ndjson", blockRange[0], blockRange[len(blockRange)-1]) - data := []byte(eventsND) + var walletEvents bytes.Buffer - err = e.SaveToS3(&key, &data) + for _, e := range wallet { + b, err := json.Marshal(e) if err != nil { - panic(err) + return err } - e.Print("saved %s \n", key) - } -} -func sliceRange(start, end, part int64) [][]int64 { - var result [][]int64 - step := (end - start) / part - for i := int64(0); i < part; i++ { - var arr []int64 - if i == 0 { - arr = append(arr, start) - arr = append(arr, start+step) - } else if i == part-1 { - arr = append(arr, start+i*step+1) - arr = append(arr, end) - } else { - arr = append(arr, start+i*step+1) - arr = append(arr, start+(i+1)*step) - } - result = append(result, arr) + walletEvents.Write(b) + walletEvents.WriteByte('\n') } - return result -} -func (e *EthereumCrawler) CurrentPrice(currency string, coin ChainType) (Price, error) { - if currency == "" { - currency = "USD" + if c.walletBucket[len(c.walletBucket)-1] == '/' { + c.walletBucket = c.walletBucket[:len(c.walletBucket)-1] } - if len(currency) != 3 { - return Price{}, fmt.Errorf("invalid currency") + dest := fmt.Sprintf("%s/%s/block=%d.ndjson", Ethereum.String(), c.Network.String(), block) + + err := c.S3.UploadBytes(c.walletBucket, dest, &walletEvents) + + if err != nil { + return err } - var price Price - var m map[string]interface{} + c.Logger.Info("uploaded block %d to %s\n", block, dest) + return nil +} + +func (c *EthereumCrawler) Close() { + c.Wg.Wait() + c.Elapsed = time.Since(c.Begin) + c.Client.Close() + close(c.Sema) + c.Logger.Info("events consumed=%d\n", c.EventsConsumed) + c.Logger.Info("time elapsed=%s\n", c.Elapsed.Round(time.Millisecond)) +} - url := fmt.Sprintf("https://min-api.cryptocompare.com/data/price?fsym=%s&tsyms=%s", coin.Short(), currency) +func (c *EthereumCrawler) ProcessBlock(height int) ([]*Event, []*WalletEvent, error) { + l := c.Logger - price.Coin = coin.Short() - price.Currency = currency - price.Time = time.Now().UTC() + var events []*Event + var walletEvents []*WalletEvent - req, err := e.HttpClient.Get(url) + block, err := c.Client.BlockByNumber(context.Background(), big.NewInt(int64(height))) if err != nil { - return price, err + return nil, nil, err } - defer req.Body.Close() + l.Info("processing block=%d\n", block.Number().Int64()) - res, err := io.ReadAll(req.Body) + blockEvent, err := c.EventFromBlock(block) if err != nil { - return price, err + return nil, nil, err } - body := bytes.NewReader(res) + events = append(events, blockEvent) - err = json.NewDecoder(body).Decode(&m) + if block.Transactions().Len() > 0 { + for i, tx := range block.Transactions() { + l.Info("processing tx %d of %d in block %d\n", i+1, block.Transactions().Len(), block.Number().Int64()) - if err != nil { - return price, err - } + receipt, err := c.Client.TransactionReceipt(context.Background(), tx.Hash()) - v, ok := m["USD"] + if err != nil { + return nil, nil, err + } - if !ok { - return price, fmt.Errorf("invalid response") - } + txEvents, walletEvent, err := c.EventsFromTransaction(block, receipt) - price.Value = v.(float64) + if err != nil { + return nil, nil, err + } - return price, nil + events = append(events, txEvents...) + walletEvents = append(walletEvents, walletEvent...) + } + } + return events, walletEvents, nil } -func (c ChainType) Short() string { - switch c { - case Ethereum: - return "ETH" - case Iotex: - return "IOTX" +func (c *EthereumCrawler) EventsFromTransaction(b *types.Block, receipt *types.Receipt) ([]*Event, []*WalletEvent, error) { + var txEvents []*Event + var walletEvents []*WalletEvent + + l := c.Logger + + for index, tx := range b.Transactions() { + txEvent := Event{ + Chain: Ethereum, + Network: c.Network, + Provider: Casimir, + Block: b.Hash().Hex(), + Type: Transaction, + Height: int64(b.Number().Uint64()), + Transaction: tx.Hash().Hex(), + ReceivedAt: time.Unix(int64(b.Time()), 0).Format("2006-01-02 15:04:05.999999999"), + } + + if tx.Value() != nil { + txEvent.Amount = tx.Value().String() + } - default: - panic("invalid chain type") + txEvent.GasFee = new(big.Int).Mul(tx.GasPrice(), big.NewInt(int64(receipt.GasUsed))).String() + + if tx.To() != nil { + txEvent.Recipient = tx.To().Hex() + recipeintBalance, err := c.Client.BalanceAt(context.Background(), *tx.To(), b.Number()) + + if err != nil { + return nil, nil, err + } + + txEvent.RecipientBalance = recipeintBalance.String() + } + + sender, err := c.Client.TransactionSender(context.Background(), tx, b.Hash(), uint(index)) + + if err != nil { + return nil, nil, err + } + + if sender.Hex() != "" { + txEvent.Sender = sender.Hex() + + senderBalance, err := c.Client.BalanceAt(context.Background(), sender, b.Number()) + + if err != nil { + return nil, nil, err + } + + txEvent.SenderBalance = senderBalance.String() + } + + txEvents = append(txEvents, &txEvent) + + senderWalletEvent := WalletEvent{ + WalletAddress: txEvent.Sender, + Balance: txEvent.SenderBalance, + Direction: Outgoing, + TxId: txEvent.Transaction, + ReceivedAt: txEvent.ReceivedAt, + Amount: txEvent.Amount, + Price: txEvent.Price, + GasFee: txEvent.GasFee, + } + + walletEvents = append(walletEvents, &senderWalletEvent) + + receiptWalletEvent := WalletEvent{ + WalletAddress: txEvent.Recipient, + Balance: txEvent.RecipientBalance, + Direction: Incoming, + TxId: txEvent.Transaction, + ReceivedAt: txEvent.ReceivedAt, + Amount: txEvent.Amount, + Price: txEvent.Price, + GasFee: txEvent.GasFee, + } + walletEvents = append(walletEvents, &receiptWalletEvent) + // TODO: handle contract events (staking action) + } + + if len(walletEvents) == 0 || len(walletEvents) != len(txEvents)*2 { + l.Error("wallet events and tx events mismatch, wallet events=%d tx events=%d", len(walletEvents), len(txEvents)) } + + return txEvents, walletEvents, nil } -func (e *EthereumCrawler) Print(s string, args ...interface{}) { - if e.Verbose { - fmt.Printf(s, args...) +func (c *EthereumCrawler) EventFromBlock(b *types.Block) (*Event, error) { + event := Event{ + Chain: Ethereum, + Network: c.Network, + Provider: Casimir, + Type: Block, + Height: int64(b.Number().Uint64()), + Block: b.Hash().Hex(), + ReceivedAt: time.Unix(int64(b.Time()), 0).Format("2006-01-02 15:04:05.999999999"), } + return &event, nil } -func SaveErrorLog(data []byte) error { - file := fmt.Sprintf("%s.log", "crawler") +func (c *EthereumCrawler) Introspect() error { + l := c.Logger + + l.Info("introspecting...\n") - f, err := os.OpenFile(file, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + err := c.Glue.LoadDatabases() if err != nil { return err } - defer f.Close() + err = c.Glue.LoadTables(AnalyticsDatabaseDev) - if _, err := f.Write(data); err != nil { + if err != nil { return err } - return nil -} + for _, t := range c.Glue.Tables { + tableVersion, err := strconv.Atoi(string([]rune(*t.Name)[len(*t.Name)-1])) -func (e *EthereumCrawler) SaveToS3(key *string, data *[]byte) error { - if len(*data) == 0 { - return fmt.Errorf("data cannot be empty") - } + if err != nil { + return err + } - bucket := os.Getenv("EVENT_BUCKET") + table := Table{ + Database: *t.DatabaseName, + Name: *t.Name, + Version: strconv.Itoa(tableVersion), + } - _, err := e.S3Client.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(bucket), - Key: key, - Body: bytes.NewReader(*data), - }) + resourceVersion, err := ResourceVersion() - if err != nil { - return err + if err != nil { + return err + } + + // we expect table version to match resource version otherwise the resoure is not ready yet wait + if tableVersion != resourceVersion { + l.Error(fmt.Sprintf("database=%s %s table=%s resource_version=%s \n", AnalyticsDatabaseDev, table.String(), *t.Name, strconv.Itoa(resourceVersion))) + return errors.New("resource version does not match table version") + } + + if t.StorageDescriptor.Location != nil { + table.Bucket = *t.StorageDescriptor.Location + } + + if t.StorageDescriptor.SerdeInfo.Name == nil { + serde := t.StorageDescriptor.SerdeInfo.SerializationLibrary + table.SerDe = strings.Split(*serde, ".")[3] + } else { + table.SerDe = *t.StorageDescriptor.SerdeInfo.Name + } + + if strings.Contains(*t.Name, "event") { + c.txBucket = table.Bucket + } else if strings.Contains(*t.Name, "staking") { + c.stakingBucket = table.Bucket + } else if strings.Contains(*t.Name, "wallet") { + c.walletBucket = table.Bucket + } + } + + if strings.HasPrefix(c.txBucket, "s3://") { + c.txBucket = strings.TrimPrefix(c.txBucket, "s3://") + } + + if strings.HasPrefix(c.walletBucket, "s3://") { + c.walletBucket = strings.TrimPrefix(c.walletBucket, "s3://") } + return nil } -func (e *EthereumCrawler) ListS3Files() ([]string, error) { - bucket := os.Getenv("EVENT_BUCKET") +func ResourceVersion() (int, error) { + f, err := os.ReadFile("common/data/package.json") - if bucket == "" { - return nil, errors.New("bucket name not set") + if err != nil { + return 0, err } - prefix := "" + var pkg PkgJSON + + err = json.Unmarshal(f, &pkg) - input := &s3.ListObjectsInput{ - Bucket: &bucket, - Prefix: &prefix, + if err != nil { + return 0, err } - result, err := e.S3Client.ListObjects(input) + var major int - if err != nil { - return nil, err + semver := strings.Split(pkg.Version, ".") + + if len(semver) < 3 { + return 0, errors.New("invalid semver") } - var files []string + major, err = strconv.Atoi(semver[0]) - for _, object := range result.Contents { - files = append(files, *object.Key) + if err != nil { + return 0, err + } + if major < 1 { + return 0, errors.New("major version must be greater than 0") } - return files, nil + return major, nil +} + +func (t Table) String() string { + return fmt.Sprintf("table=%s version=%s database=%s bucket=%s serde=%s", t.Name, t.Version, t.Database, t.Bucket, t.SerDe) } diff --git a/services/crawler/crawler_test.go b/services/crawler/crawler_test.go new file mode 100644 index 000000000..9b86f24b4 --- /dev/null +++ b/services/crawler/crawler_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "context" + "testing" +) + +func TestNewEthereumCrawler(t *testing.T) { + var err error + crawler, err := NewEthereumCrawler() + + if err != nil { + t.Error(err) + } + + _, err = crawler.Client.BlockNumber(context.Background()) + + if err != nil { + t.Error(err) + } +} + +func TestEthereumCrawler_Introspect(t *testing.T) { + crawler, err := NewEthereumCrawler() + + if err != nil { + t.Error(err) + } + + err = crawler.Introspect() + + if err != nil { + t.Error(err) + } + + if crawler.txBucket == "" { + t.Error("introspection returned no tables, expected events table") + } + + if crawler.walletBucket == "" { + t.Error("introspection returned no tables, expected wallets table") + } + + if crawler.stakingBucket == "" { + t.Error("introspection returned no tables, expected staking action table") + } +} diff --git a/services/crawler/ethereum.go b/services/crawler/ethereum.go new file mode 100644 index 000000000..bd2b71d90 --- /dev/null +++ b/services/crawler/ethereum.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/url" + "time" + + "github.com/ethereum/go-ethereum/ethclient" +) + +type ChainType string +type NetworkType string +type ProviderType string +type EventType string + +const ( + Ethereum ChainType = "ethereum" + Bitcoin ChainType = "bitcoin" + Iotex ChainType = "iotex" + + EtheruemMainnet NetworkType = "mainnet" + EtheruemGoerli NetworkType = "goerli" + + Casimir ProviderType = "casimir" + + Block EventType = "block" + Transaction EventType = "transaction" + Contract EventType = "contract" +) + +type EthereumClient struct { + Client *ethclient.Client + Network NetworkType + Provider ProviderType + Url url.URL +} + +func NewEthereumClient(Provider ProviderType, url url.URL) (*EthereumClient, error) { + if url.String() == "" { + return nil, errors.New("etheruem rpc url is empty") + } + + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(3*time.Second)) + + defer cancel() + + client, err := ethclient.DialContext(ctx, url.String()) + + if err != nil { + return nil, err + } + + defer cancel() + + var net NetworkType + + id, err := client.NetworkID(ctx) + + if err != nil { + return nil, err + } + + switch id.Int64() { + case 1: + net = EtheruemMainnet + case 5: + net = EtheruemGoerli + default: + return nil, fmt.Errorf("unsupported network id: %d", id.Int64()) + } + + return &EthereumClient{ + Client: client, + Network: net, + Provider: Casimir, + Url: url, + }, nil +} + +func (c ChainType) String() string { + switch c { + case Ethereum: + return "ethereum" + case Bitcoin: + return "bitcoin" + case Iotex: + return "iotex" + default: + return "" + } +} + +func (c NetworkType) String() string { + switch c { + case EtheruemMainnet: + return "mainnet" + case EtheruemGoerli: + return "goerli" + default: + return "" + } +} diff --git a/services/crawler/ethereum_test.go b/services/crawler/ethereum_test.go new file mode 100644 index 000000000..f6545039d --- /dev/null +++ b/services/crawler/ethereum_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "context" + "net/url" + "os" + "testing" +) + +func TestNewEtheruemClient(t *testing.T) { + err := LoadEnv() + + if err != nil { + t.Fatal(err) + } + + raw := os.Getenv("ETHEREUM_RPC") + + url, err := url.Parse(raw) + + if err != nil { + t.Fatal(err) + } + + client, err := NewEthereumClient(Casimir, *url) + + if err != nil { + t.Fatal(err) + } + + head, err := client.Client.HeaderByNumber(context.Background(), nil) + + if err != nil { + t.Fatal(err) + } + + t.Log(head.Number.String()) +} diff --git a/services/crawler/event.go b/services/crawler/event.go deleted file mode 100644 index 288833ee0..000000000 --- a/services/crawler/event.go +++ /dev/null @@ -1,154 +0,0 @@ -package main - -import ( - "encoding/json" - "time" - - "github.com/ethereum/go-ethereum/core/types" - "github.com/xitongsys/parquet-go/parquet" - "github.com/xitongsys/parquet-go/writer" -) - -type ChainType string -type NetworkType string -type ProviderType string -type EventType string - -const ( - Ethereum ChainType = "ethereum" - Iotex ChainType = "iotex" - - Mainnet NetworkType = "mainnet" - Testnet NetworkType = "testnet" - - Consensus ProviderType = "consensus" - Alchemy ProviderType = "alchemy" - - Block EventType = "block" - Transaction EventType = "transaction" -) - -type Price struct { - Value float64 `json:"price,omitempty"` - Currency string `json:"currency,omitempty"` - Coin string `json:"coin,omitempty"` - Time time.Time `json:"time,omitempty"` -} - -type Event struct { - Chain ChainType `json:"chain,omitempty"` - Network NetworkType `json:"network,omitempty"` - Provider ProviderType `json:"provider,omitempty"` - Type EventType `json:"type,omitempty"` - Height int64 `json:"height,omitempty"` - Block string `json:"block,omitempty"` - Transaction string `json:"transaction,omitempty"` - ReceivedAt string `json:"received_at,omitempty"` - Sender string `json:"sender,omitempty"` - Recipient string `json:"recipient,omitempty"` - Amount int64 `json:"amount,omitempty"` - Price float64 `json:"price,omitempty"` - SenderBalance int64 `json:"sender_balance,omitempty"` - RecipientBalance int64 `json:"recipient_balance,omitempty"` -} - -func NewParquetWriter(schema []*parquet.SchemaElement) (*writer.ParquetWriter, error) { - by := "casimir.parquet.schema" - - w := writer.ParquetWriter{ - NP: 4, - Footer: &parquet.FileMetaData{ - Schema: schema, - Version: 1, - CreatedBy: &by, - }, - RowGroupSize: 128 * 1024 * 1024, - PageSize: 8 * 1024, - } - w.CompressionType = parquet.CompressionCodec_SNAPPY - return &w, nil -} - -func NewBlockEvent(block *types.Block) (Event, error) { - event := Event{ - Chain: Ethereum, - Network: Mainnet, - Provider: Alchemy, - Type: Block, - Height: block.Number().Int64(), - Block: block.Hash().Hex(), - ReceivedAt: time.Unix(int64(block.Time()), 0).Format("2006-01-02 15:04:05.999999999"), - } - return event, nil -} - -func NewTxEvent(block *types.Block, tx *types.Transaction) Event { - event := Event{ - Network: Mainnet, - Provider: Alchemy, - Chain: Ethereum, - Type: Transaction, - Block: tx.Hash().Hex(), - Transaction: tx.Hash().Hex(), - Height: block.Number().Int64(), - ReceivedAt: time.Unix(int64(block.Time()), 0).Format("2006-01-02 15:04:05.999999999"), - } - - if tx.To() != nil { - event.Recipient = tx.To().Hex() - } - - if tx.Value() != nil { - event.Amount = tx.Value().Int64() - } - - return event -} - -func NDJSON(events *[]Event) (string, error) { - var ndjson []byte - - for _, event := range *events { - line, err := json.Marshal(event) - - if err != nil { - return "", err - } - line = append(line, '\r') - ndjson = append(ndjson, line...) - } - return string(ndjson), nil -} - -func (c ChainType) String() string { - switch c { - case Ethereum: - return "ethereum" - case Iotex: - return "iotex" - default: - panic("unknown") - } -} - -func (p ProviderType) String() string { - switch p { - case Alchemy: - return "alchemy" - case Consensus: - return "consensus" - default: - panic("unknown") - } -} - -func (n NetworkType) String() string { - switch n { - case Testnet: - return "testnet" - case Mainnet: - return "mainnet" - default: - panic("unknown") - } -} diff --git a/services/crawler/exchange.go b/services/crawler/exchange.go new file mode 100644 index 000000000..7e5cdfb6b --- /dev/null +++ b/services/crawler/exchange.go @@ -0,0 +1,284 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "time" +) + +type Currency string + +const ( + USD Currency = "USD" +) + +// Exchange is an interface for getting current and historical prices +// for a given coin and currency. Supports multiple exchanges. +type Exchange interface { + // CurrentPrice returns the current price of a coin + CurrentPrice(coin ChainType, currency Currency) (ExchangePrice, error) + // GetHistoricalPrice returns the price of a coin at a given time + HistoricalPrice(coin ChainType, currency Currency, received time.Time) (ExchangePrice, error) +} + +type ExchangePrice struct { + Value float64 `json:"price,omitempty"` + Currency string `json:"currency,omitempty"` + Coin string `json:"coin,omitempty"` + Time time.Time `json:"time,omitempty"` +} + +type CryptoCompareExchange struct { + BaseUrl string + ApiKey string + Version int +} + +type CryptoCompareTickerResponse struct { + Data struct { + ETHUSD struct { + Type string `json:"TYPE"` + Market string `json:"MARKET"` + Instrument string `json:"INSTRUMENT"` + Ccseq int `json:"CCSEQ"` + Value float64 `json:"VALUE"` + ValueFlag string `json:"VALUE_FLAG"` + ValueLastUpdateTs int `json:"VALUE_LAST_UPDATE_TS"` + ValueLastUpdateNs int `json:"VALUE_LAST_UPDATE_NS"` + LastUpdateQuantity float64 `json:"LAST_UPDATE_QUANTITY"` + LastUpdateQuoteQuantity float64 `json:"LAST_UPDATE_QUOTE_QUANTITY"` + LastUpdateCcseq int `json:"LAST_UPDATE_CCSEQ"` + CurrentHourVolume float64 `json:"CURRENT_HOUR_VOLUME"` + CurrentHourQuoteVolume float64 `json:"CURRENT_HOUR_QUOTE_VOLUME"` + CurrentHourOpen float64 `json:"CURRENT_HOUR_OPEN"` + CurrentHourHigh float64 `json:"CURRENT_HOUR_HIGH"` + CurrentHourLow float64 `json:"CURRENT_HOUR_LOW"` + CurrentHourTotalIndexUpdates int `json:"CURRENT_HOUR_TOTAL_INDEX_UPDATES"` + CurrentHourChange float64 `json:"CURRENT_HOUR_CHANGE"` + CurrentHourChangePercentage float64 `json:"CURRENT_HOUR_CHANGE_PERCENTAGE"` + CurrentDayVolume float64 `json:"CURRENT_DAY_VOLUME"` + CurrentDayQuoteVolume float64 `json:"CURRENT_DAY_QUOTE_VOLUME"` + CurrentDayOpen float64 `json:"CURRENT_DAY_OPEN"` + CurrentDayHigh float64 `json:"CURRENT_DAY_HIGH"` + CurrentDayLow float64 `json:"CURRENT_DAY_LOW"` + CurrentDayTotalIndexUpdates int `json:"CURRENT_DAY_TOTAL_INDEX_UPDATES"` + CurrentDayChange float64 `json:"CURRENT_DAY_CHANGE"` + CurrentDayChangePercentage float64 `json:"CURRENT_DAY_CHANGE_PERCENTAGE"` + CurrentWeekVolume float64 `json:"CURRENT_WEEK_VOLUME"` + CurrentWeekQuoteVolume float64 `json:"CURRENT_WEEK_QUOTE_VOLUME"` + CurrentWeekOpen float64 `json:"CURRENT_WEEK_OPEN"` + CurrentWeekHigh float64 `json:"CURRENT_WEEK_HIGH"` + CurrentWeekLow float64 `json:"CURRENT_WEEK_LOW"` + CurrentWeekTotalIndexUpdates int `json:"CURRENT_WEEK_TOTAL_INDEX_UPDATES"` + CurrentWeekChange float64 `json:"CURRENT_WEEK_CHANGE"` + CurrentWeekChangePercentage float64 `json:"CURRENT_WEEK_CHANGE_PERCENTAGE"` + CurrentMonthVolume float64 `json:"CURRENT_MONTH_VOLUME"` + CurrentMonthQuoteVolume float64 `json:"CURRENT_MONTH_QUOTE_VOLUME"` + CurrentMonthOpen float64 `json:"CURRENT_MONTH_OPEN"` + CurrentMonthHigh float64 `json:"CURRENT_MONTH_HIGH"` + CurrentMonthLow float64 `json:"CURRENT_MONTH_LOW"` + CurrentMonthTotalIndexUpdates int `json:"CURRENT_MONTH_TOTAL_INDEX_UPDATES"` + CurrentMonthChange float64 `json:"CURRENT_MONTH_CHANGE"` + CurrentMonthChangePercentage float64 `json:"CURRENT_MONTH_CHANGE_PERCENTAGE"` + CurrentYearVolume float64 `json:"CURRENT_YEAR_VOLUME"` + CurrentYearQuoteVolume float64 `json:"CURRENT_YEAR_QUOTE_VOLUME"` + CurrentYearOpen float64 `json:"CURRENT_YEAR_OPEN"` + CurrentYearHigh float64 `json:"CURRENT_YEAR_HIGH"` + CurrentYearLow float64 `json:"CURRENT_YEAR_LOW"` + CurrentYearTotalIndexUpdates int `json:"CURRENT_YEAR_TOTAL_INDEX_UPDATES"` + CurrentYearChange float64 `json:"CURRENT_YEAR_CHANGE"` + CurrentYearChangePercentage float64 `json:"CURRENT_YEAR_CHANGE_PERCENTAGE"` + Moving24HourVolume float64 `json:"MOVING_24_HOUR_VOLUME"` + Moving24HourQuoteVolume float64 `json:"MOVING_24_HOUR_QUOTE_VOLUME"` + Moving24HourOpen float64 `json:"MOVING_24_HOUR_OPEN"` + Moving24HourHigh float64 `json:"MOVING_24_HOUR_HIGH"` + Moving24HourLow float64 `json:"MOVING_24_HOUR_LOW"` + Moving24HourTotalIndexUpdates int `json:"MOVING_24_HOUR_TOTAL_INDEX_UPDATES"` + Moving24HourChange float64 `json:"MOVING_24_HOUR_CHANGE"` + Moving24HourChangePercentage float64 `json:"MOVING_24_HOUR_CHANGE_PERCENTAGE"` + Moving7DayVolume float64 `json:"MOVING_7_DAY_VOLUME"` + Moving7DayQuoteVolume float64 `json:"MOVING_7_DAY_QUOTE_VOLUME"` + Moving7DayOpen float64 `json:"MOVING_7_DAY_OPEN"` + Moving7DayHigh float64 `json:"MOVING_7_DAY_HIGH"` + Moving7DayLow float64 `json:"MOVING_7_DAY_LOW"` + Moving7DayTotalIndexUpdates int `json:"MOVING_7_DAY_TOTAL_INDEX_UPDATES"` + Moving7DayChange float64 `json:"MOVING_7_DAY_CHANGE"` + Moving7DayChangePercentage float64 `json:"MOVING_7_DAY_CHANGE_PERCENTAGE"` + Moving30DayVolume float64 `json:"MOVING_30_DAY_VOLUME"` + Moving30DayQuoteVolume float64 `json:"MOVING_30_DAY_QUOTE_VOLUME"` + Moving30DayOpen float64 `json:"MOVING_30_DAY_OPEN"` + Moving30DayHigh float64 `json:"MOVING_30_DAY_HIGH"` + Moving30DayLow float64 `json:"MOVING_30_DAY_LOW"` + Moving30DayTotalIndexUpdates int `json:"MOVING_30_DAY_TOTAL_INDEX_UPDATES"` + Moving30DayChange float64 `json:"MOVING_30_DAY_CHANGE"` + Moving30DayChangePercentage float64 `json:"MOVING_30_DAY_CHANGE_PERCENTAGE"` + Moving90DayVolume float64 `json:"MOVING_90_DAY_VOLUME"` + Moving90DayQuoteVolume float64 `json:"MOVING_90_DAY_QUOTE_VOLUME"` + Moving90DayOpen float64 `json:"MOVING_90_DAY_OPEN"` + Moving90DayHigh float64 `json:"MOVING_90_DAY_HIGH"` + Moving90DayLow float64 `json:"MOVING_90_DAY_LOW"` + Moving90DayTotalIndexUpdates int `json:"MOVING_90_DAY_TOTAL_INDEX_UPDATES"` + Moving90DayChange float64 `json:"MOVING_90_DAY_CHANGE"` + Moving90DayChangePercentage float64 `json:"MOVING_90_DAY_CHANGE_PERCENTAGE"` + Moving180DayVolume float64 `json:"MOVING_180_DAY_VOLUME"` + Moving180DayQuoteVolume float64 `json:"MOVING_180_DAY_QUOTE_VOLUME"` + Moving180DayOpen float64 `json:"MOVING_180_DAY_OPEN"` + Moving180DayHigh float64 `json:"MOVING_180_DAY_HIGH"` + Moving180DayLow float64 `json:"MOVING_180_DAY_LOW"` + Moving180DayTotalIndexUpdates int `json:"MOVING_180_DAY_TOTAL_INDEX_UPDATES"` + Moving180DayChange float64 `json:"MOVING_180_DAY_CHANGE"` + Moving180DayChangePercentage float64 `json:"MOVING_180_DAY_CHANGE_PERCENTAGE"` + Moving365DayVolume float64 `json:"MOVING_365_DAY_VOLUME"` + Moving365DayQuoteVolume float64 `json:"MOVING_365_DAY_QUOTE_VOLUME"` + Moving365DayOpen float64 `json:"MOVING_365_DAY_OPEN"` + Moving365DayHigh float64 `json:"MOVING_365_DAY_HIGH"` + Moving365DayLow float64 `json:"MOVING_365_DAY_LOW"` + Moving365DayTotalIndexUpdates int `json:"MOVING_365_DAY_TOTAL_INDEX_UPDATES"` + Moving365DayChange float64 `json:"MOVING_365_DAY_CHANGE"` + Moving365DayChangePercentage float64 `json:"MOVING_365_DAY_CHANGE_PERCENTAGE"` + LifetimeFirstUpdateTs int `json:"LIFETIME_FIRST_UPDATE_TS"` + LifetimeVolume float64 `json:"LIFETIME_VOLUME"` + LifetimeQuoteVolume float64 `json:"LIFETIME_QUOTE_VOLUME"` + LifetimeOpen float64 `json:"LIFETIME_OPEN"` + LifetimeHigh float64 `json:"LIFETIME_HIGH"` + LifetimeHighTs int `json:"LIFETIME_HIGH_TS"` + LifetimeLow float64 `json:"LIFETIME_LOW"` + LifetimeLowTs int `json:"LIFETIME_LOW_TS"` + LifetimeTotalIndexUpdates int `json:"LIFETIME_TOTAL_INDEX_UPDATES"` + LifetimeChange float64 `json:"LIFETIME_CHANGE"` + LifetimeChangePercentage float64 `json:"LIFETIME_CHANGE_PERCENTAGE"` + } `json:"ETH-USD"` + } `json:"Data"` + Err struct { + } `json:"Err"` +} + +func NewHttpClientWithTimeout(time time.Duration) (*http.Client, error) { + client := &http.Client{ + Timeout: time, + } + return client, nil +} + +func NewCryptoCompareExchange(apiKey string) (Exchange, error) { + if apiKey == "" { + return nil, errors.New("api key is required") + } + return CryptoCompareExchange{ + ApiKey: apiKey, + BaseUrl: "https://data-api.cryptocompare.com", // v2 api + }, nil +} + +func (c CryptoCompareExchange) CurrentPrice(coin ChainType, currency Currency) (ExchangePrice, error) { + price := ExchangePrice{ + Coin: coin.Short(), + Currency: currency.String(), + Time: time.Now(), + } + + httpClient, err := NewHttpClientWithTimeout(time.Duration(2 * time.Second)) + + if err != nil { + return price, err + } + + path := "index/cc/v1/latest/tick" + + url := fmt.Sprintf("%s/%s?market=ccix&instruments=%s-%s", c.BaseUrl, path, coin.Short(), currency.String()) + + req, err := httpClient.Get(url) + + if err != nil { + return price, err + } + + defer req.Body.Close() + + res, err := io.ReadAll(req.Body) + + if err != nil { + return price, err + } + + var response CryptoCompareTickerResponse + + err = json.Unmarshal(res, &response) + + if err != nil { + return price, err + } + + price.Value = response.Data.ETHUSD.CurrentDayOpen + return price, nil +} + +func (c CryptoCompareExchange) HistoricalPrice(coin ChainType, currency Currency, received time.Time) (ExchangePrice, error) { + price := ExchangePrice{ + Coin: coin.Short(), + Currency: currency.String(), + Time: received, + } + + httpClient, err := NewHttpClientWithTimeout(time.Duration(2 * time.Second)) + + if err != nil { + return price, err + } + + path := "index/cc/v1/historical/minutes" + + limit := 20 + + fmt.Println(received) + // conver the time.Time to unix timestamp + unixTime := received.UTC().Unix() + + url := fmt.Sprintf("%s/%s?market=ccix&instrument=%s-%s&limit=%d&to_ts=%d", c.BaseUrl, path, coin.Short(), currency.String(), limit, unixTime) + + fmt.Println(unixTime) + fmt.Println(url) + + req, err := httpClient.Get(url) + + if err != nil { + return price, err + } + + defer req.Body.Close() + + res, err := io.ReadAll(req.Body) + + if err != nil { + return price, err + } + + var response CryptoCompareTickerResponse + + err = json.Unmarshal(res, &response) + + if err != nil { + return price, err + } + + price.Value = response.Data.ETHUSD.CurrentDayOpen + return price, nil +} + +func (c ChainType) Short() string { + switch c { + case Ethereum: + return "ETH" + case Bitcoin: + return "BTC" + default: + return "" + } +} + +func (c Currency) String() string { + return string(c) +} diff --git a/services/crawler/exchange_test.go b/services/crawler/exchange_test.go new file mode 100644 index 000000000..83f4307ea --- /dev/null +++ b/services/crawler/exchange_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "fmt" + "math/big" + "net/url" + "os" + "testing" +) + +func TestCurrentPrice(t *testing.T) { + err := LoadEnv() + + if err != nil { + t.Error(err) + } + + key := os.Getenv("CRYPTOCOMPARE_API_KEY") + + exchange, err := NewCryptoCompareExchange(key) + + if err != nil { + t.Error(err) + } + + price, err := exchange.CurrentPrice(Ethereum, USD) + + if err != nil { + t.Error(err) + } + + fmt.Println(price) +} + +func TestHistoricalPrice(t *testing.T) { + err := LoadEnv() + + if err != nil { + t.Error(err) + } + + raw := os.Getenv("ETHEREUM_RPC") + + fmt.Println(raw) + + url, err := url.Parse(raw) + + if err != nil { + t.Fatal(err) + } + + client, err := NewEthereumClient(Casimir, *url) + + fmt.Println(client) + + if err != nil { + t.Fatal(err) + } + + block, err := client.Client.BlockByNumber(context.Background(), big.NewInt(10000000)) + + if err != nil { + t.Error(err) + } + + fmt.Println(block.Number().String()) + + // if err != nil { + // t.Error(err) + // } + + // key := os.Getenv("CRYPTOCOMPARE_API_KEY") + + // exchange, err := NewCryptoCompareExchange(key) + + // if err != nil { + // t.Error(err) + // } + + // price, err := exchange.HistoricalPrice(Ethereum, USD, block.ReceivedAt) + + // if err != nil { + // t.Error(err) + // } + + // fmt.Println(price) +} diff --git a/services/crawler/glue.go b/services/crawler/glue.go new file mode 100644 index 000000000..c81220e18 --- /dev/null +++ b/services/crawler/glue.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/glue" + "github.com/aws/aws-sdk-go-v2/service/glue/types" +) + +const ( + AnalyticsDatabaseDev = "casimir_analytics_database_dev" + AnalyticsDatabaseProd = "casimir_analytics_database_prod" +) + +type GlueClient struct { + Client *glue.Client + Databases []types.Database + Tables []types.Table +} + +func LoadDefaultAWSConfig() (*aws.Config, error) { + region := "us-east-2" + config, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion(region), + ) + + if err != nil { + return nil, err + } + + return &config, nil +} + +func NewGlueClient(config *aws.Config) (*GlueClient, error) { + client := glue.NewFromConfig(*config) + + return &GlueClient{ + Client: client, + }, nil +} + +func (g *GlueClient) LoadDatabases() error { + req, err := g.Client.GetDatabases(context.Background(), &glue.GetDatabasesInput{}) + + if err != nil { + return err + } + + g.Databases = append(g.Databases, req.DatabaseList...) + + if req.NextToken != nil { + err := g.LoadDatabases() + + if err != nil { + return err + } + } + + return nil +} + +func (g *GlueClient) LoadTables(databaseName string) error { + input := &glue.GetTablesInput{ + DatabaseName: aws.String(databaseName), + } + + req, err := g.Client.GetTables(context.Background(), input) + + if err != nil { + return err + } + + g.Tables = append(g.Tables, req.TableList...) + + if req.NextToken != nil { + input.NextToken = req.NextToken + err = g.LoadTables(databaseName) + + if err != nil { + return err + } + } + + return nil +} diff --git a/services/crawler/glue_test.go b/services/crawler/glue_test.go new file mode 100644 index 000000000..5bd6f5695 --- /dev/null +++ b/services/crawler/glue_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "fmt" + "strconv" + "strings" + "testing" +) + +func TestNewGlueClient(t *testing.T) { + config, err := LoadDefaultAWSConfig() + + if err != nil { + t.Error(err) + } + + client, err := NewGlueClient(config) + + if err != nil { + t.Error(err) + } + + if client == nil { + t.Error("client is nil") + } +} + +func TestGlueClient_LoadDatabases(t *testing.T) { + config, err := LoadDefaultAWSConfig() + + if err != nil { + t.Error(err) + } + + glue, err := NewGlueClient(config) + + if err != nil { + t.Error(err) + } + + err = glue.LoadDatabases() + + if err != nil { + t.Error(err) + } + + if len(glue.Databases) == 0 { + t.Error("no databases returned") + } + + for _, database := range glue.Databases { + if *database.Name == AnalyticsDatabaseDev { + return + } + } + t.Error("database not found") +} + +func TestGlueClient_LoadTables(t *testing.T) { + config, err := LoadDefaultAWSConfig() + + if err != nil { + t.Error(err) + } + + client, err := NewGlueClient(config) + + if err != nil { + t.Error(err) + } + + err = client.LoadTables(AnalyticsDatabaseDev) + + if err != nil { + t.Error(err) + } + + for _, table := range client.Tables { + name := strings.Split(*table.Name, "")[len(*table.Name)-1] + + resourceVersion, err := strconv.Atoi(name) + + if err != nil { + t.Error(err) + } + + if resourceVersion < 1 { + fmt.Println(name) + t.Error("resource version must be greater than 0") + } + } +} diff --git a/services/crawler/go.mod b/services/crawler/go.mod index a9fce3d94..a145adaa2 100644 --- a/services/crawler/go.mod +++ b/services/crawler/go.mod @@ -1,40 +1,51 @@ -module github.com/hawyar/candle +module github.com/consensusnetworks/crawler go 1.19 require ( - github.com/aws/aws-sdk-go v1.44.188 - github.com/ethereum/go-ethereum v1.10.26 - github.com/joho/godotenv v1.4.0 - github.com/segmentio/go-athena v0.0.0-20181208004937-dfa5f1818930 - github.com/urfave/cli/v2 v2.24.1 - github.com/xitongsys/parquet-go v1.6.2 + github.com/aws/aws-sdk-go-v2 v1.18.1 + github.com/aws/aws-sdk-go-v2/config v1.1.1 + github.com/aws/aws-sdk-go-v2/service/glue v1.51.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 + github.com/ethereum/go-ethereum v1.12.0 + github.com/fatih/color v1.15.0 + github.com/joho/godotenv v1.5.1 + github.com/schollz/progressbar v1.0.0 + github.com/urfave/cli/v2 v2.25.7 ) require ( github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect - github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect - github.com/apache/thrift v0.14.2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.1.1 // indirect + github.com/aws/smithy-go v1.13.5 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/deckarep/golang-set v1.8.0 // indirect + github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/go-ole/go-ole v1.2.1 // indirect - github.com/go-stack/stack v1.8.0 // indirect - github.com/golang/snappy v0.0.4 // indirect + github.com/go-stack/stack v1.8.1 // indirect github.com/gorilla/websocket v1.4.2 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/klauspost/compress v1.13.1 // indirect - github.com/pierrec/lz4/v4 v4.1.8 // indirect + github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/satori/go.uuid v1.2.0 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect - github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect - golang.org/x/sys v0.1.0 // indirect - golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect + golang.org/x/crypto v0.1.0 // indirect + golang.org/x/sys v0.7.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect ) diff --git a/services/crawler/go.sum b/services/crawler/go.sum index 576b24c7b..7b5e483f8 100644 --- a/services/crawler/go.sum +++ b/services/crawler/go.sum @@ -1,395 +1,156 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= -github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 h1:byKBBF2CKWBjjA4J1ZL2JXttJULvWSl50LegTyRZ728= -github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= -github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.14.2 h1:hY4rAyg7Eqbb27GB6gkhUKrRAuc8xRjlNtJq+LseKeY= -github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= -github.com/aws/aws-sdk-go v1.44.188 h1:NCN6wFDWKU72Ka+f7cCk3HRj1KxkEXhRdr7lO8oBRRQ= -github.com/aws/aws-sdk-go v1.44.188/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= +github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= +github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= +github.com/aws/aws-sdk-go-v2/config v1.1.1 h1:ZAoq32boMzcaTW9bcUacBswAmHTbvlvDJICgHFZuECo= +github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= +github.com/aws/aws-sdk-go-v2/credentials v1.1.1 h1:NbvWIM1Mx6sNPTxowHgS2ewXCRp+NGTzUYb/96FZJbY= +github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2 h1:EtEU7WRaWliitZh2nmuxEXrN0Cb8EgPUFGIoTMeqbzI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2/go.mod h1:3hGg3PpiEjHnrkrlasTfxFqUsZ2GCk/fMUn4CbKgSkM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26 h1:wscW+pnn3J1OYnanMnza5ZVYXLX4cKk5rAvUAl4Qu+c= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.26/go.mod h1:MtYiox5gvyB+OyP0Mr0Sm/yzbEAIPL9eijj/ouHAPw0= +github.com/aws/aws-sdk-go-v2/service/glue v1.51.0 h1:ezO4k5Nnowpedvk1LGtYQUWiXaWx+zq+CCmbcTcZBoI= +github.com/aws/aws-sdk-go-v2/service/glue v1.51.0/go.mod h1:wMCE0B6l8eHb57l2DMYCGxt0rHIfcu3RvIY7SAfc+Fs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 h1:y2+VQzC6Zh2ojtV2LoC0MNwHWc6qXv/j2vrQtlftkdA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11/go.mod h1:iV4q2hsqtNECrfmlXyord9u4zyuFEJX9eLgLpSPzWA8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29 h1:zZSLP3v3riMOP14H7b4XP0uyfREDQOYv2cqIrvTXDNQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.29/go.mod h1:z7EjRjVwZ6pWcWdI2H64dKttvzaP99jRIj5hphW0M5U= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2/go.mod h1:45MfaXZ0cNbeuT0KQ1XJylq8A6+OpVV2E5kvY/Kq+u8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3 h1:dBL3StFxHtpBzJJ/mNEsjXVgfO+7jR0dAIEwLqMapEA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.3/go.mod h1:f1QyiAsvIv4B49DmCqrhlXqyaR+0IxMmyX+1P+AnzOM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0 h1:ya7fmrN2fE7s1P2gaPbNg5MTkERVWfsH8ToP1YC4Z9o= +github.com/aws/aws-sdk-go-v2/service/s3 v1.35.0/go.mod h1:aVbf0sko/TsLWHx30c/uVu7c62+0EAJ3vbxaJga0xCw= +github.com/aws/aws-sdk-go-v2/service/sso v1.1.1 h1:37QubsarExl5ZuCBlnRP+7l1tNwZPBSTqpTBrPH98RU= +github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= +github.com/aws/aws-sdk-go-v2/service/sts v1.1.1 h1:TJoIfnIFubCX0ACVeJ0w46HEH5MwjwYN4iFhuYIhfIY= +github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= +github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= +github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= -github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= +github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= +github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.10.26 h1:i/7d9RBBwiXCEuyduBQzJw/mKmnvzsN14jqBmytw72s= -github.com/ethereum/go-ethereum v1.10.26/go.mod h1:EYFyF19u3ezGLD4RqOkLq+ZCXzYbLoNDdZlMt7kyKFg= +github.com/ethereum/go-ethereum v1.12.0 h1:bdnhLPtqETd4m3mS8BGMNvBTf36bO5bx/hxE2zljOa0= +github.com/ethereum/go-ethereum v1.12.0/go.mod h1:/oo2X/dZLJjf2mJ6YT9wcWxa4nNJDBKDBU6sFIpx1Gs= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A= -github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= -github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= -github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= +github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8= +github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= -github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= -github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ= -github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= -github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4= -github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= -github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/segmentio/go-athena v0.0.0-20181208004937-dfa5f1818930 h1:Tn2Ryh7e9oN9TK19Y0vP/1Rfr1/DqBxXC8qrWKS4ez8= -github.com/segmentio/go-athena v0.0.0-20181208004937-dfa5f1818930/go.mod h1:umGD11uSGUY8Vd0lbo1jJUEAk4FxVV3YE5wRSXx1Lbk= +github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM= +github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= +github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= -github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= -github.com/urfave/cli/v2 v2.24.1 h1:/QYYr7g0EhwXEML8jO+8OYt5trPnLHS0p3mrgExJ5NU= -github.com/urfave/cli/v2 v2.24.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= -github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= -github.com/xitongsys/parquet-go v1.6.2 h1:MhCaXii4eqceKPu9BwrjLqyK10oX9WF+xGhwvwbw7xM= -github.com/xitongsys/parquet-go v1.6.2/go.mod h1:IulAQyalCm0rPiZVNnCgm/PCL64X2tdSVGMQ/UeKqWA= -github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= -github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k= -github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= +github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/services/crawler/logger.go b/services/crawler/logger.go new file mode 100644 index 000000000..4dc219c96 --- /dev/null +++ b/services/crawler/logger.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + + "github.com/fatih/color" +) + +const ( + info = "[INFO]" + warn = "[WARN]" + err = "[ERROR]" +) + +type Logger interface { + Info(format string, args ...interface{}) + Warn(format string, args ...interface{}) + Error(format string, args ...interface{}) +} + +type StdoutLogger struct{} + +func NewStdoutLogger() *StdoutLogger { + return &StdoutLogger{} +} + +func (l *StdoutLogger) Info(format string, args ...interface{}) { + fmt.Printf(color.GreenString(info)+" "+format, args...) +} + +func (l *StdoutLogger) Warn(format string, args ...interface{}) { + fmt.Printf(color.YellowString(warn)+" "+format, args...) +} + +func (l *StdoutLogger) Error(format string, args ...interface{}) { + fmt.Printf(color.RedString(err)+" "+format, args...) +} diff --git a/services/crawler/main.go b/services/crawler/main.go deleted file mode 100644 index 12236979e..000000000 --- a/services/crawler/main.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "os" - - "github.com/joho/godotenv" -) - -func main() { - err := godotenv.Load() - - if err != nil { - panic(err) - } - - err = Run(os.Args) - - if err != nil { - panic(err) - } - os.Exit(0) -} diff --git a/services/crawler/s3.go b/services/crawler/s3.go new file mode 100644 index 000000000..67e317346 --- /dev/null +++ b/services/crawler/s3.go @@ -0,0 +1,96 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "os" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type S3Client struct { + Client *s3.Client +} + +func NewS3Client() (*S3Client, error) { + s3c := &S3Client{} + + cfg, err := LoadDefaultAWSConfig() + + if err != nil { + return nil, fmt.Errorf("failed to load default config for s3 client: %v", err) + } + + s3c.Client = s3.NewFromConfig(*cfg) + + return s3c, nil +} + +func (s *S3Client) Upload(bucket string, key string, fpath string) error { + var err error + + file, err := os.Open(fpath) + + if err != nil { + return fmt.Errorf("failed to open file %q: %v", fpath, err) + } + + defer file.Close() + + opt := &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: file, + } + + _, err = s.Client.PutObject(context.Background(), opt) + + if err != nil { + return fmt.Errorf("failed to put object: %v", err) + } + return nil +} + +func (s *S3Client) UploadBytes(bucket string, key string, data *bytes.Buffer) error { + opt := &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: bytes.NewReader(data.Bytes()), + } + + _, err := s.Client.PutObject(context.Background(), opt) + + if err != nil { + return fmt.Errorf("failed to put object: %v", err) + } + return nil +} + +func (s *S3Client) MultipartUpload(bucket, key, fpath string) error { + return s.Upload(bucket, key, fpath) +} + +func (s *S3Client) Get(bucket, key string) (*bytes.Buffer, error) { + var err error + + opt := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + } + + result, err := s.Client.GetObject(context.Background(), opt) + + if err != nil { + return nil, fmt.Errorf("failed to get object: %v", err) + } + + defer result.Body.Close() + + buf := new(bytes.Buffer) + + buf.ReadFrom(result.Body) + + return buf, nil +} diff --git a/services/crawler/streamer.go b/services/crawler/streamer.go index 898f6e31a..dc0baf336 100644 --- a/services/crawler/streamer.go +++ b/services/crawler/streamer.go @@ -1,273 +1,286 @@ package main import ( - "bytes" "context" - "encoding/json" - "fmt" - "io" - "log" - "net/http" + "errors" + "math/big" + "net/url" + "os" + "sync" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/xitongsys/parquet-go/writer" + "github.com/schollz/progressbar" ) type EthereumStreamer struct { - BaseConfig - EthClient *ethclient.Client - S3Client *s3.S3 - HttpClient *http.Client - ParquetWriter *writer.ParquetWriter + EthereumClient + Logger + Mutex *sync.Mutex + Begin time.Time + Elapsed time.Duration + Glue *GlueClient + S3 *S3Client + Exchange Exchange + Head uint64 + // the ever increasing curren block number of the chain + ProcessingBlock uint64 + EventsConsumed int + Version int + Progress *progressbar.ProgressBar } -func NewEthereumStreamer(config BaseConfig) (*EthereumStreamer, error) { - s := &EthereumStreamer{ - BaseConfig: config, - } - - ethClient, err := NewEthereumClient(config.Url) +func NewEthereumStreamer() (*EthereumStreamer, error) { + err := LoadEnv() if err != nil { return nil, err } - s.EthClient = ethClient + raw := os.Getenv("ETHEREUM_WS") + + if raw == "" { + return nil, errors.New("ETHERUEM_WS env variable is not set") + } - s3Client, err := NewS3Client() + url, err := url.Parse(raw) if err != nil { return nil, err } - s.S3Client = s3Client - - httpClient, err := NewHTTPClient() + client, err := NewEthereumClient(Casimir, *url) if err != nil { return nil, err } - s.HttpClient = httpClient - return s, nil -} - -func (e *EthereumStreamer) Stream() error { - fmt.Println("streaming ethereum blocks...") - - header := make(chan *types.Header) - - sub, err := e.EthClient.SubscribeNewHead(context.Background(), header) + head, err := client.Client.BlockNumber(context.Background()) if err != nil { - return err + return nil, err } - defer sub.Unsubscribe() - if err != nil { - return err + return nil, err } - for { - select { - case err := <-sub.Err(): - return err - case h := <-header: + key := os.Getenv("CRYPTOCOMPARE_API_KEY") - fmt.Printf("block: %d", h.Number.Int64()) + exchange, err := NewCryptoCompareExchange(key) - var events []Event + if err != nil { + return nil, err + } - block, err := e.EthClient.BlockByHash(context.Background(), h.Hash()) + return &EthereumStreamer{ + EthereumClient: *client, + Logger: NewStdoutLogger(), + Mutex: &sync.Mutex{}, + Head: head, + Exchange: exchange, + Begin: time.Now(), + }, nil +} - if err != nil { - return err - } +func (s *EthereumStreamer) Subscribe(ctx context.Context) <-chan *types.Header { + l := s.Logger - blockEvent, err := NewBlockEvent(block) + header := make(chan *types.Header) - if err != nil { - return err - } + producer, err := s.Client.SubscribeNewHead(ctx, header) - price, err := e.CurrentPrice("USD", Ethereum) + if err != nil { + l.Error(err.Error()) + close(header) + return header + } - if err != nil { - return err + go func() { + defer close(header) + defer producer.Unsubscribe() + + for { + select { + case <-ctx.Done(): + return + case err := <-producer.Err(): + l.Error(err.Error()) + close(header) + return } + } + }() + return header +} - blockEvent.Price = price.Value - - events = append(events, blockEvent) - - if len(block.Transactions()) > 0 { - for _, t := range block.Transactions() { - tx, pending, err := e.EthClient.TransactionByHash(context.Background(), t.Hash()) - - if err != nil { - return err - } - - if pending { - fmt.Printf("skipping pending transaction: %s", tx.Hash().String()) - continue - } - - if err != nil { - return err - } - - if tx.To() == nil { - fmt.Printf("skipping contract transaction: %s", tx.Hash().String()) - continue - } - - chainID, err := e.EthClient.NetworkID(context.Background()) +func (s *EthereumStreamer) Stream() error { + l := s.Logger - if err != nil { - return err - } + l.Info("starting streamer\n") - txMsg, err := tx.AsMessage(types.NewEIP155Signer(chainID), nil) + ctx := context.Background() - if err != nil { - return err - } + head := s.Subscribe(ctx) - from := txMsg.From() - to := txMsg.To() + for { + select { + case <-ctx.Done(): + return nil + case h := <-head: + _, _, err := s.ProcessBlock(int(h.Number.Uint64())) - txEvent := NewTxEvent(block, tx) + if err != nil { + l.Error(err.Error()) + return err + } + } + } +} - fromBalance, err := e.EthClient.BalanceAt(context.Background(), from, nil) +func (s *EthereumStreamer) ProcessBlock(height int) ([]*Event, []*WalletEvent, error) { + l := s.Logger - if err != nil { - return err - } + var events []*Event + var walletEvents []*WalletEvent - toBalance, err := e.EthClient.BalanceAt(context.Background(), *to, nil) + block, err := s.Client.BlockByNumber(context.Background(), big.NewInt(int64(height))) - if err != nil { - return nil - } + if err != nil { + return nil, nil, err + } - txEvent.Sender = from.Hex() - txEvent.SenderBalance = fromBalance.Int64() + l.Info("processing block=%d\n", block.Number().Int64()) - txEvent.Recipient = to.Hex() - txEvent.RecipientBalance = toBalance.Int64() - txEvent.Price = price.Value + blockEvent, err := s.EventFromBlock(block) - events = append(events, txEvent) - } - } + if err != nil { + return nil, nil, err + } - if err != nil { - return err - } + events = append(events, blockEvent) - key := fmt.Sprintf("%d.json", blockEvent.Height) + if block.Transactions().Len() > 0 { + for i, tx := range block.Transactions() { + l.Info("processing tx %d of %d in block %d\n", i+1, block.Transactions().Len(), block.Number().Int64()) - ev, err := NDJSON(&events) + receipt, err := s.Client.TransactionReceipt(context.Background(), tx.Hash()) if err != nil { - return err + return nil, nil, err } - err = e.Save(key, []byte(ev)) + txEvents, walletEvent, err := s.EventsFromTransaction(block, receipt) if err != nil { - return err + return nil, nil, err } - fmt.Printf("\t saved %d events \n", len(events)) + events = append(events, txEvents...) + walletEvents = append(walletEvents, walletEvent...) } } + return events, walletEvents, nil } -func (e *EthereumStreamer) CurrentPrice(currency string, coin ChainType) (Price, error) { - if currency == "" { - currency = "USD" - } +func (s *EthereumStreamer) EventsFromTransaction(b *types.Block, receipt *types.Receipt) ([]*Event, []*WalletEvent, error) { + var txEvents []*Event + var walletEvents []*WalletEvent + + l := s.Logger + + for index, tx := range b.Transactions() { + txEvent := Event{ + Chain: Ethereum, + Network: s.Network, + Provider: Casimir, + Block: b.Hash().Hex(), + Type: Transaction, + Height: int64(b.Number().Uint64()), + Transaction: tx.Hash().Hex(), + ReceivedAt: time.Unix(int64(b.Time()), 0).Format("2006-01-02 15:04:05.999999999"), + } - if len(currency) != 3 { - return Price{}, fmt.Errorf("invalid currency") - } + if tx.Value() != nil { + txEvent.Amount = tx.Value().String() + } - var price Price - var m map[string]interface{} + txEvent.GasFee = new(big.Int).Mul(tx.GasPrice(), big.NewInt(int64(receipt.GasUsed))).String() - url := fmt.Sprintf("https://min-api.cryptocompare.com/data/price?fsym=%s&tsyms=%s", coin.Short(), currency) + if tx.To() != nil { + txEvent.Recipient = tx.To().Hex() + recipeintBalance, err := s.Client.BalanceAt(context.Background(), *tx.To(), b.Number()) - price.Coin = coin.Short() - price.Currency = currency - price.Time = time.Now().UTC() + if err != nil { + return nil, nil, err + } - req, err := e.HttpClient.Get(url) + txEvent.RecipientBalance = recipeintBalance.String() + } - if err != nil { - return price, err - } + sender, err := s.Client.TransactionSender(context.Background(), tx, b.Hash(), uint(index)) - defer func(Body io.ReadCloser) { - err := Body.Close() if err != nil { - log.Fatal(err) + return nil, nil, err } - }(req.Body) - - res, err := io.ReadAll(req.Body) - - if err != nil { - return price, err - } - body := bytes.NewReader(res) + if sender.Hex() != "" { + txEvent.Sender = sender.Hex() - err = json.NewDecoder(body).Decode(&m) + senderBalance, err := s.Client.BalanceAt(context.Background(), sender, b.Number()) - if err != nil { - return price, err - } - - v, ok := m["USD"] - - if !ok { - return price, fmt.Errorf("invalid response") - } - - price.Value = v.(float64) + if err != nil { + return nil, nil, err + } - return price, nil -} + txEvent.SenderBalance = senderBalance.String() + } -func (e *EthereumStreamer) Save(dest string, data []byte) error { - b := "casimir-etl-event-bucket-dev" + txEvents = append(txEvents, &txEvent) + + senderWalletEvent := WalletEvent{ + WalletAddress: txEvent.Sender, + Balance: txEvent.SenderBalance, + Direction: Outgoing, + TxId: txEvent.Transaction, + ReceivedAt: txEvent.ReceivedAt, + Amount: txEvent.Amount, + Price: txEvent.Price, + GasFee: txEvent.GasFee, + } - input := &s3.PutObjectInput{ - Body: bytes.NewReader(data), - Bucket: aws.String(b), - Key: aws.String(dest), + walletEvents = append(walletEvents, &senderWalletEvent) + + receiptWalletEvent := WalletEvent{ + WalletAddress: txEvent.Recipient, + Balance: txEvent.RecipientBalance, + Direction: Incoming, + TxId: txEvent.Transaction, + ReceivedAt: txEvent.ReceivedAt, + Amount: txEvent.Amount, + Price: txEvent.Price, + GasFee: txEvent.GasFee, + } + walletEvents = append(walletEvents, &receiptWalletEvent) + // TODO: handle contract events (staking action) } - _, err := e.S3Client.PutObject(input) - - if err != nil { - return err + if len(walletEvents) == 0 || len(walletEvents) != len(txEvents)*2 { + l.Error("wallet events and tx events mismatch, wallet events=%d tx events=%d", len(walletEvents), len(txEvents)) } - return nil + return txEvents, walletEvents, nil } -func (e Event) String() string { - b, err := json.Marshal(e) - if err != nil { - return "" +func (s *EthereumStreamer) EventFromBlock(b *types.Block) (*Event, error) { + event := Event{ + Chain: Ethereum, + Network: s.Network, + Provider: Casimir, + Type: Block, + Height: int64(b.Number().Uint64()), + Block: b.Hash().Hex(), + ReceivedAt: time.Unix(int64(b.Time()), 0).Format("2006-01-02 15:04:05.999999999"), } - return string(b) + return &event, nil } diff --git a/services/crawler/streamer_test.go b/services/crawler/streamer_test.go new file mode 100644 index 000000000..2feff31e6 --- /dev/null +++ b/services/crawler/streamer_test.go @@ -0,0 +1,20 @@ +package main + +import ( + "context" + "testing" +) + +func TestNewEthereumStreamer(t *testing.T) { + streamer, err := NewEthereumStreamer() + + if err != nil { + t.Error(err) + } + + _, err = streamer.Client.BlockNumber(context.Background()) + + if err != nil { + t.Error(err) + } +}