From bef89fda660d56169606e32da4f371a990685a4c Mon Sep 17 00:00:00 2001 From: Mudit Gupta Date: Sat, 14 Nov 2020 22:02:50 +0530 Subject: [PATCH 1/4] cmd/faucet, cmd/puppeth: support for twitter API --- cmd/faucet/faucet.go | 88 +++++++++++++++++++++++++++++------- cmd/puppeth/module_faucet.go | 9 +++- cmd/puppeth/wizard_faucet.go | 23 ++++++++++ 3 files changed, 103 insertions(+), 17 deletions(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 346c412acb6e..59274f26a874 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -83,6 +83,8 @@ var ( noauthFlag = flag.Bool("noauth", false, "Enables funding requests without authentication") logFlag = flag.Int("loglevel", 3, "Log level to use for Ethereum and the faucet") + + twitterBearerToken = flag.String("twitter.token", "", "Twitter bearer token to authenticate with the twitter API") ) var ( @@ -443,9 +445,10 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } // Retrieve the Ethereum address to fund, the requesting user and a profile picture var ( - username string - avatar string - address common.Address + unique_id string + username string + avatar string + address common.Address ) switch { case strings.HasPrefix(msg.URL, "https://gist.github.com/"): @@ -462,11 +465,13 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } continue case strings.HasPrefix(msg.URL, "https://twitter.com/"): - username, avatar, address, err = authTwitter(msg.URL) + unique_id, username, avatar, address, err = authTwitter(msg.URL, *twitterBearerToken) case strings.HasPrefix(msg.URL, "https://www.facebook.com/"): username, avatar, address, err = authFacebook(msg.URL) + unique_id = username case *noauthFlag: username, avatar, address, err = authNoAuth(msg.URL) + unique_id = username default: //lint:ignore ST1005 This error is to be displayed in the browser err = errors.New("Something funky happened, please open an issue at https://github.com/ethereum/go-ethereum/issues") @@ -486,7 +491,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { fund bool timeout time.Time ) - if timeout = f.timeouts[username]; time.Now().After(timeout) { + if timeout = f.timeouts[unique_id]; time.Now().After(timeout) { // User wasn't funded recently, create the funding transaction amount := new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether) amount = new(big.Int).Mul(amount, new(big.Int).Exp(big.NewInt(5), big.NewInt(int64(msg.Tier)), nil)) @@ -520,7 +525,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { timeout := time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute grace := timeout / 288 // 24h timeout => 5m grace - f.timeouts[username] = time.Now().Add(timeout - grace) + f.timeouts[unique_id] = time.Now().Add(timeout - grace) fund = true } f.lock.Unlock() @@ -684,23 +689,74 @@ func sendSuccess(conn *websocket.Conn, msg string) error { } // authTwitter tries to authenticate a faucet request using Twitter posts, returning -// the username, avatar URL and Ethereum address to fund on success. -func authTwitter(url string) (string, string, common.Address, error) { +// the uniqueness identifier (user id/username), username, avatar URL and Ethereum address to fund on success. +func authTwitter(url string, token string) (string, string, string, common.Address, error) { // Ensure the user specified a meaningful URL, no fancy nonsense parts := strings.Split(url, "/") if len(parts) < 4 || parts[len(parts)-2] != "status" { //lint:ignore ST1005 This error is to be displayed in the browser - return "", "", common.Address{}, errors.New("Invalid Twitter status URL") + return "", "", "", common.Address{}, errors.New("Invalid Twitter status URL") + } + + // Twitter's API isn't really friendly with direct links. + // It is restricted to 300 queries / 15 minute with an app api key. + // Anything more will require read only authorization from the users and that we want to avoid. + + // If twitter bearer token is provided, use the twitter api + if token != "" { + // Strip any query parameters from the tweet id + tweetID := strings.Split(parts[len(parts)-1], "?")[0] + + // Query the tweet details from Twitter + url := "https://api.twitter.com/2/tweets/" + tweetID + "?expansions=author_id&user.fields=profile_image_url" + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", "", "", common.Address{}, err + } + req.Header.Set("Authorization", "Bearer "+token) + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", "", "", common.Address{}, err + } + defer res.Body.Close() + + var result struct { + Data struct { + AuthorID string `json:"author_id"` + ID string `json:"id"` + Text string `json:"text"` + } `json:"data"` + Includes struct { + Users []struct { + ProfileImageURL string `json:"profile_image_url"` + Username string `json:"username"` + ID string `json:"id"` + Name string `json:"name"` + } `json:"users"` + } `json:"includes"` + } + + err = json.NewDecoder(res.Body).Decode(&result) + if err != nil { + return "", "", "", common.Address{}, err + } + + address := common.HexToAddress(regexp.MustCompile("0x[0-9a-fA-F]{40}").FindString(result.Data.Text)) + if address == (common.Address{}) { + //lint:ignore ST1005 This error is to be displayed in the browser + return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund") + } + return result.Data.AuthorID + "@twitter", result.Includes.Users[0].Username, result.Includes.Users[0].ProfileImageURL, address, nil } - // Twitter's API isn't really friendly with direct links. Still, we don't - // want to do ask read permissions from users, so just load the public posts + + // Twiter API token isn't provided so we just load the public posts // and scrape it for the Ethereum address and profile URL. We need to load // the mobile page though since the main page loads tweet contents via JS. url = strings.Replace(url, "https://twitter.com/", "https://mobile.twitter.com/", 1) res, err := http.Get(url) if err != nil { - return "", "", common.Address{}, err + return "", "", "", common.Address{}, err } defer res.Body.Close() @@ -708,24 +764,24 @@ func authTwitter(url string) (string, string, common.Address, error) { parts = strings.Split(res.Request.URL.String(), "/") if len(parts) < 4 || parts[len(parts)-2] != "status" { //lint:ignore ST1005 This error is to be displayed in the browser - return "", "", common.Address{}, errors.New("Invalid Twitter status URL") + return "", "", "", common.Address{}, errors.New("Invalid Twitter status URL") } username := parts[len(parts)-3] body, err := ioutil.ReadAll(res.Body) if err != nil { - return "", "", common.Address{}, err + return "", "", "", common.Address{}, err } address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body))) if address == (common.Address{}) { //lint:ignore ST1005 This error is to be displayed in the browser - return "", "", common.Address{}, errors.New("No Ethereum address found to fund") + return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund") } var avatar string if parts = regexp.MustCompile("src=\"([^\"]+twimg.com/profile_images[^\"]+)\"").FindStringSubmatch(string(body)); len(parts) == 2 { avatar = parts[1] } - return username + "@twitter", avatar, address, nil + return username + "@twitter", username + "@twitter", avatar, address, nil } // authFacebook tries to authenticate a faucet request using Facebook posts, diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go index 987bed14aa57..2527fc971c74 100644 --- a/cmd/puppeth/module_faucet.go +++ b/cmd/puppeth/module_faucet.go @@ -46,6 +46,7 @@ ENTRYPOINT [ \ "--faucet.name", "{{.FaucetName}}", "--faucet.amount", "{{.FaucetAmount}}", "--faucet.minutes", "{{.FaucetMinutes}}", "--faucet.tiers", "{{.FaucetTiers}}", \ "--account.json", "/account.json", "--account.pass", "/account.pass" \ {{if .CaptchaToken}}, "--captcha.token", "{{.CaptchaToken}}", "--captcha.secret", "{{.CaptchaSecret}}"{{end}}{{if .NoAuth}}, "--noauth"{{end}} \ + {{if .TwitterToken}}, "--twitter.token", "{{.TwitterToken}}", ]` // faucetComposefile is the docker-compose.yml file required to deploy and maintain @@ -70,7 +71,8 @@ services: - FAUCET_MINUTES={{.FaucetMinutes}} - FAUCET_TIERS={{.FaucetTiers}} - CAPTCHA_TOKEN={{.CaptchaToken}} - - CAPTCHA_SECRET={{.CaptchaSecret}} + - CAPTCHA_SECRET={{.CaptchaSecret}} + - TWITTER_TOKEN={{.TwitterToken}} - NO_AUTH={{.NoAuth}}{{if .VHost}} - VIRTUAL_HOST={{.VHost}} - VIRTUAL_PORT=8080{{end}} @@ -103,6 +105,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config "FaucetMinutes": config.minutes, "FaucetTiers": config.tiers, "NoAuth": config.noauth, + "TwitterToken": config.twitterToken, }) files[filepath.Join(workdir, "Dockerfile")] = dockerfile.Bytes() @@ -120,6 +123,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config "FaucetMinutes": config.minutes, "FaucetTiers": config.tiers, "NoAuth": config.noauth, + "TwitterToken": config.twitterToken, }) files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes() @@ -152,6 +156,7 @@ type faucetInfos struct { noauth bool captchaToken string captchaSecret string + twitterToken string } // Report converts the typed struct into a plain string->string map, containing @@ -165,6 +170,7 @@ func (info *faucetInfos) Report() map[string]string { "Funding cooldown (base tier)": fmt.Sprintf("%d mins", info.minutes), "Funding tiers": strconv.Itoa(info.tiers), "Captha protection": fmt.Sprintf("%v", info.captchaToken != ""), + "Using Twitter API": fmt.Sprintf("%v", info.twitterToken != ""), "Ethstats username": info.node.ethstats, } if info.noauth { @@ -243,5 +249,6 @@ func checkFaucet(client *sshClient, network string) (*faucetInfos, error) { captchaToken: infos.envvars["CAPTCHA_TOKEN"], captchaSecret: infos.envvars["CAPTCHA_SECRET"], noauth: infos.envvars["NO_AUTH"] == "true", + twitterToken: infos.envvars["TWITTER_TOKEN"], }, nil } diff --git a/cmd/puppeth/wizard_faucet.go b/cmd/puppeth/wizard_faucet.go index 9f753ad68bb9..47e05cd9c106 100644 --- a/cmd/puppeth/wizard_faucet.go +++ b/cmd/puppeth/wizard_faucet.go @@ -102,6 +102,29 @@ func (w *wizard) deployFaucet() { infos.captchaSecret = w.readPassword() } } + + // Accessing the twitter api requires a bearer token, request it + if infos.twitterToken != "" { + fmt.Println() + fmt.Println("Reuse previous twitter API Bearer token (y/n)? (default = yes)") + if !w.readDefaultYesNo(true) { + infos.twitterToken = "" + } + } + if infos.twitterToken == "" { + // No previous twitter token (or old one discarded) + fmt.Println() + fmt.Println("Enable twitter API (y/n)? (default = no)") + if !w.readDefaultYesNo(false) { + log.Warn("The faucet will fallback to using direct calls") + } else { + // Twitter api explicitly requested, read the bearer token + fmt.Println() + fmt.Printf("What is the twitter API Bearer token?\n") + infos.twitterToken = w.readString() + } + } + // Figure out where the user wants to store the persistent data fmt.Println() if infos.node.datadir == "" { From 159b7c663275a81ec20e0d13dbf0ae8837e6ae08 Mon Sep 17 00:00:00 2001 From: Mudit Gupta Date: Sun, 15 Nov 2020 10:24:49 +0530 Subject: [PATCH 2/4] cmd/faucet, cmd/puppeth: refactoring from review suggestions --- cmd/faucet/faucet.go | 92 +++++++++++++++++++----------------- cmd/puppeth/module_faucet.go | 4 +- 2 files changed, 51 insertions(+), 45 deletions(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 59274f26a874..0e3f9dab4c8c 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -704,49 +704,7 @@ func authTwitter(url string, token string) (string, string, string, common.Addre // If twitter bearer token is provided, use the twitter api if token != "" { - // Strip any query parameters from the tweet id - tweetID := strings.Split(parts[len(parts)-1], "?")[0] - - // Query the tweet details from Twitter - url := "https://api.twitter.com/2/tweets/" + tweetID + "?expansions=author_id&user.fields=profile_image_url" - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return "", "", "", common.Address{}, err - } - req.Header.Set("Authorization", "Bearer "+token) - res, err := http.DefaultClient.Do(req) - if err != nil { - return "", "", "", common.Address{}, err - } - defer res.Body.Close() - - var result struct { - Data struct { - AuthorID string `json:"author_id"` - ID string `json:"id"` - Text string `json:"text"` - } `json:"data"` - Includes struct { - Users []struct { - ProfileImageURL string `json:"profile_image_url"` - Username string `json:"username"` - ID string `json:"id"` - Name string `json:"name"` - } `json:"users"` - } `json:"includes"` - } - - err = json.NewDecoder(res.Body).Decode(&result) - if err != nil { - return "", "", "", common.Address{}, err - } - - address := common.HexToAddress(regexp.MustCompile("0x[0-9a-fA-F]{40}").FindString(result.Data.Text)) - if address == (common.Address{}) { - //lint:ignore ST1005 This error is to be displayed in the browser - return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund") - } - return result.Data.AuthorID + "@twitter", result.Includes.Users[0].Username, result.Includes.Users[0].ProfileImageURL, address, nil + return authTwitterWithToken(parts[len(parts)-1], token) } // Twiter API token isn't provided so we just load the public posts @@ -784,6 +742,54 @@ func authTwitter(url string, token string) (string, string, string, common.Addre return username + "@twitter", username + "@twitter", avatar, address, nil } +// authTwitterWithToken tries to authenticate a faucet request using Twitter's API, returning +// the uniqueness identifier (user id/username), username, avatar URL and Ethereum address to fund on success. +func authTwitterWithToken(tweetID string, token string) (string, string, string, common.Address, error) { + // Strip any query parameters from the tweet id + sanitizedTweetID := strings.Split(tweetID, "?")[0] + + // Query the tweet details from Twitter + url := fmt.Sprintf("https://api.twitter.com/2/tweets/%s?expansions=author_id&user.fields=profile_image_url", sanitizedTweetID) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", "", "", common.Address{}, err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + res, err := http.DefaultClient.Do(req) + if err != nil { + return "", "", "", common.Address{}, err + } + defer res.Body.Close() + + var result struct { + Data struct { + AuthorID string `json:"author_id"` + ID string `json:"id"` + Text string `json:"text"` + } `json:"data"` + Includes struct { + Users []struct { + ProfileImageURL string `json:"profile_image_url"` + Username string `json:"username"` + ID string `json:"id"` + Name string `json:"name"` + } `json:"users"` + } `json:"includes"` + } + + err = json.NewDecoder(res.Body).Decode(&result) + if err != nil { + return "", "", "", common.Address{}, err + } + + address := common.HexToAddress(regexp.MustCompile("0x[0-9a-fA-F]{40}").FindString(result.Data.Text)) + if address == (common.Address{}) { + //lint:ignore ST1005 This error is to be displayed in the browser + return "", "", "", common.Address{}, errors.New("No Ethereum address found to fund") + } + return result.Data.AuthorID + "@twitter", result.Includes.Users[0].Username, result.Includes.Users[0].ProfileImageURL, address, nil +} + // authFacebook tries to authenticate a faucet request using Facebook posts, // returning the username, avatar URL and Ethereum address to fund on success. func authFacebook(url string) (string, string, common.Address, error) { diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go index 2527fc971c74..2527e137f2c6 100644 --- a/cmd/puppeth/module_faucet.go +++ b/cmd/puppeth/module_faucet.go @@ -71,8 +71,8 @@ services: - FAUCET_MINUTES={{.FaucetMinutes}} - FAUCET_TIERS={{.FaucetTiers}} - CAPTCHA_TOKEN={{.CaptchaToken}} - - CAPTCHA_SECRET={{.CaptchaSecret}} - - TWITTER_TOKEN={{.TwitterToken}} + - CAPTCHA_SECRET={{.CaptchaSecret}} + - TWITTER_TOKEN={{.TwitterToken}} - NO_AUTH={{.NoAuth}}{{if .VHost}} - VIRTUAL_HOST={{.VHost}} - VIRTUAL_PORT=8080{{end}} From 3c8444c33ed30ed0b66776c71d631f3c672b0833 Mon Sep 17 00:00:00 2001 From: Mudit Gupta Date: Mon, 16 Nov 2020 15:57:50 +0530 Subject: [PATCH 3/4] cmd/faucet: minor refactoring --- cmd/faucet/faucet.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 0e3f9dab4c8c..4a3b68365327 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -445,10 +445,10 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } // Retrieve the Ethereum address to fund, the requesting user and a profile picture var ( - unique_id string - username string - avatar string - address common.Address + id string + username string + avatar string + address common.Address ) switch { case strings.HasPrefix(msg.URL, "https://gist.github.com/"): @@ -465,13 +465,13 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } continue case strings.HasPrefix(msg.URL, "https://twitter.com/"): - unique_id, username, avatar, address, err = authTwitter(msg.URL, *twitterBearerToken) + id, username, avatar, address, err = authTwitter(msg.URL, *twitterBearerToken) case strings.HasPrefix(msg.URL, "https://www.facebook.com/"): username, avatar, address, err = authFacebook(msg.URL) - unique_id = username + id = username case *noauthFlag: username, avatar, address, err = authNoAuth(msg.URL) - unique_id = username + id = username default: //lint:ignore ST1005 This error is to be displayed in the browser err = errors.New("Something funky happened, please open an issue at https://github.com/ethereum/go-ethereum/issues") @@ -491,7 +491,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { fund bool timeout time.Time ) - if timeout = f.timeouts[unique_id]; time.Now().After(timeout) { + if timeout = f.timeouts[id]; time.Now().After(timeout) { // User wasn't funded recently, create the funding transaction amount := new(big.Int).Mul(big.NewInt(int64(*payoutFlag)), ether) amount = new(big.Int).Mul(amount, new(big.Int).Exp(big.NewInt(5), big.NewInt(int64(msg.Tier)), nil)) @@ -525,7 +525,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { timeout := time.Duration(*minutesFlag*int(math.Pow(3, float64(msg.Tier)))) * time.Minute grace := timeout / 288 // 24h timeout => 5m grace - f.timeouts[unique_id] = time.Now().Add(timeout - grace) + f.timeouts[id] = time.Now().Add(timeout - grace) fund = true } f.lock.Unlock() @@ -739,7 +739,7 @@ func authTwitter(url string, token string) (string, string, string, common.Addre if parts = regexp.MustCompile("src=\"([^\"]+twimg.com/profile_images[^\"]+)\"").FindStringSubmatch(string(body)); len(parts) == 2 { avatar = parts[1] } - return username + "@twitter", username + "@twitter", avatar, address, nil + return username + "@twitter", username, avatar, address, nil } // authTwitterWithToken tries to authenticate a faucet request using Twitter's API, returning From 14addfa56739b9042a145987416996af01c8abc5 Mon Sep 17 00:00:00 2001 From: Mudit Gupta Date: Tue, 24 Nov 2020 16:20:28 +0530 Subject: [PATCH 4/4] cmd/faucet: ensure numeric tweetid --- cmd/faucet/faucet.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 4a3b68365327..2026f8962a79 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -748,6 +748,11 @@ func authTwitterWithToken(tweetID string, token string) (string, string, string, // Strip any query parameters from the tweet id sanitizedTweetID := strings.Split(tweetID, "?")[0] + // Ensure numeric tweetID + if !regexp.MustCompile("^[0-9]+$").MatchString(sanitizedTweetID) { + return "", "", "", common.Address{}, errors.New("Invalid Tweet URL") + } + // Query the tweet details from Twitter url := fmt.Sprintf("https://api.twitter.com/2/tweets/%s?expansions=author_id&user.fields=profile_image_url", sanitizedTweetID) req, err := http.NewRequest("GET", url, nil)