From fb272f38562733378d5740be97218e8018e994b4 Mon Sep 17 00:00:00 2001 From: zjiajun Date: Fri, 23 Feb 2024 17:21:18 +0800 Subject: [PATCH] support cookies by account --- .gitignore | 1 + README.md | 1 + engine/engine.go | 51 ++++++++++++++++++++------------- internal/scraper/scraper.go | 8 +++--- internal/storage/diskstorage.go | 9 +++--- 5 files changed, 41 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index f4079e6..71c88d5 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ build/ /warmane dev_config.yml +*.cookies .ropeproject/ vendor/ diff --git a/README.md b/README.md index b5ca0b8..eca0417 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ + [x] 验证码识别 + [x] 自动Points签到 + [x] 多账户支持 ++ [x] 支持cookies保存,免登录 + [ ] 商城人物数据爬取 + [ ] 人物数据分析并存储 + [ ] 多维度查询 diff --git a/engine/engine.go b/engine/engine.go index ba0f09a..877e6af 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -2,6 +2,7 @@ package engine import ( "encoding/json" + "fmt" "github.com/gocolly/colly/v2" "github.com/golang/glog" "gitub.com/zJiajun/warmane/internal/captcha" @@ -31,7 +32,7 @@ const ( type Engine struct { config *config.Config - scraper *scraper.Scraper + scrapers map[string]*scraper.Scraper wg sync.WaitGroup csrfToken string } @@ -42,8 +43,8 @@ func New() *Engine { panic(err) } return &Engine{ - config: conf, - scraper: scraper.NewScraper(), + config: conf, + scrapers: make(map[string]*scraper.Scraper, len(conf.Accounts)), } } @@ -55,11 +56,20 @@ func (e *Engine) RunDailyPoints() { e.wg.Add(count) glog.Infof("开始goroutine并发处理") for _, account := range e.config.Accounts { + e.setScraper(account) go e.loginAndCollect(account) } e.wg.Wait() } +func (e *Engine) setScraper(account config.Account) { + e.scrapers[account.Username] = scraper.NewScraper(account.Username) +} + +func (e *Engine) scraper(account config.Account) *scraper.Scraper { + return e.scrapers[account.Username] +} + func (e *Engine) loginAndCollect(account config.Account) { defer e.wg.Done() if err := e.login(account); err != nil { @@ -79,11 +89,11 @@ func (e *Engine) loginAndCollect(account config.Account) { } func (e *Engine) login(account config.Account) error { - if e.existCookiesFile() { - glog.Infof("存在cookies文件,跳过登录") + if e.existCookiesFile(account) { + glog.Infof("存在[%s]cookies文件,跳过登录", account.Username) return nil } - c := e.scraper.CloneCollector() + c := e.scraper(account).CloneCollector() var err error e.csrfToken, err = getCsrfToken(c) if err != nil { @@ -102,8 +112,8 @@ func (e *Engine) login(account config.Account) error { "g-recaptcha-response": code, "userRM": "on", } - e.scraper.SetRequestHeaders(c, e.csrfToken) - e.scraper.DecodeResponse(c) + e.scraper(account).SetRequestHeaders(c, e.csrfToken) + e.scraper(account).DecodeResponse(c) var bodyMsg BodyMsg c.OnResponse(func(response *colly.Response) { bodyText := string(response.Body) @@ -127,8 +137,9 @@ func (e *Engine) login(account config.Account) error { return err } -func (e *Engine) existCookiesFile() bool { - _, err := os.Stat("www.warmane.com.cookies") +func (e *Engine) existCookiesFile(account config.Account) bool { + cookiesFile := fmt.Sprintf("www.warmane.com.%s.cookies", account.Username) + _, err := os.Stat(cookiesFile) if os.IsNotExist(err) { return false } @@ -136,16 +147,16 @@ func (e *Engine) existCookiesFile() bool { } func (e *Engine) collectPoints(account config.Account) error { - beforeCoins, beforePoints, err := e.getInfo() + beforeCoins, beforePoints, err := e.getInfo(account) if err != nil { return err } glog.Infof("账号[%s]收集签到点[前]的 coins: [%s], points: [%s]", account.Username, beforeCoins, beforePoints) - c := e.scraper.CloneCollector() - e.scraper.SetRequestHeaders(c, e.csrfToken) - e.scraper.DecodeResponse(c) + c := e.scraper(account).CloneCollector() + e.scraper(account).SetRequestHeaders(c, e.csrfToken) + e.scraper(account).DecodeResponse(c) var bodyMsg BodyMsg c.OnResponse(func(response *colly.Response) { bodyText := string(response.Body) @@ -170,7 +181,7 @@ func (e *Engine) collectPoints(account config.Account) error { if err != nil { return err } - afterCoins, afterPoints, err := e.getInfo() + afterCoins, afterPoints, err := e.getInfo(account) if err != nil { return err } @@ -180,7 +191,7 @@ func (e *Engine) collectPoints(account config.Account) error { } func (e *Engine) logout(account config.Account) error { - c := e.scraper.CloneCollector() + c := e.scraper(account).CloneCollector() err := c.Visit(config.LogoutUrl) if err != nil { return err @@ -189,10 +200,10 @@ func (e *Engine) logout(account config.Account) error { return err } -func (e *Engine) getInfo() (coins string, points string, err error) { - c := e.scraper.CloneCollector() - e.scraper.SetRequestHeaders(c, e.csrfToken) - e.scraper.DecodeResponse(c) +func (e *Engine) getInfo(account config.Account) (coins string, points string, err error) { + c := e.scraper(account).CloneCollector() + e.scraper(account).SetRequestHeaders(c, e.csrfToken) + e.scraper(account).DecodeResponse(c) c.OnHTML(config.CoinsSelector, func(element *colly.HTMLElement) { coins = element.Text }) diff --git a/internal/scraper/scraper.go b/internal/scraper/scraper.go index d4c0ce8..e0b0269 100644 --- a/internal/scraper/scraper.go +++ b/internal/scraper/scraper.go @@ -13,7 +13,7 @@ type Scraper struct { c *colly.Collector } -func NewScraper() *Scraper { +func NewScraper(name string) *Scraper { s := &Scraper{ c: colly.NewCollector( colly.AllowURLRevisit(), @@ -22,7 +22,7 @@ func NewScraper() *Scraper { ), } s.c.SetRequestTimeout(600 * time.Second) - if err := s.c.SetStorage(storage.NewDiskStorage()); err != nil { + if err := s.c.SetStorage(storage.NewDiskStorage(name)); err != nil { panic(err) } return s @@ -45,7 +45,7 @@ func (s *Scraper) SetRequestHeaders(c *colly.Collector, csrfToken string) { func (s *Scraper) DecodeResponse(c *colly.Collector) { c.OnResponse(func(response *colly.Response) { - glog.Infof("onResponse [%s], statusCode:[%d], size:[%d]", response.Request.URL, response.StatusCode, len(response.Body)) + //glog.Infof("onResponse [%s], statusCode:[%d], size:[%d]", response.Request.URL, response.StatusCode, len(response.Body)) encoding := response.Headers.Get("Content-Encoding") if encoding == "" { return @@ -56,7 +56,7 @@ func (s *Scraper) DecodeResponse(c *colly.Collector) { return } response.Body = decodeResp - glog.Infof("onResponse decode [%s] response success", response.Request.URL) + //glog.Infof("onResponse decode [%s] response success", response.Request.URL) }) } diff --git a/internal/storage/diskstorage.go b/internal/storage/diskstorage.go index 2131b51..faa3d47 100644 --- a/internal/storage/diskstorage.go +++ b/internal/storage/diskstorage.go @@ -3,7 +3,6 @@ package storage import ( "fmt" "github.com/gocolly/colly/v2/storage" - "github.com/golang/glog" "log" "net/url" "os" @@ -14,9 +13,9 @@ type DiskStorage struct { inMemoryStorage *storage.InMemoryStorage } -func NewDiskStorage() *DiskStorage { +func NewDiskStorage(name string) *DiskStorage { return &DiskStorage{ - fileName: ".cookies", + fileName: "." + name + ".cookies", inMemoryStorage: &storage.InMemoryStorage{}, } } @@ -34,7 +33,7 @@ func (ds *DiskStorage) IsVisited(requestID uint64) (bool, error) { } func (ds *DiskStorage) Cookies(u *url.URL) string { - glog.Infof("disk storage run cookies, %v", u) + //glog.Infof("disk storage run cookies, %v", u) filePath := fmt.Sprintf("%s"+ds.fileName, u.Hostname()) buf, err := os.ReadFile(filePath) if err != nil { @@ -44,7 +43,7 @@ func (ds *DiskStorage) Cookies(u *url.URL) string { } func (ds *DiskStorage) SetCookies(u *url.URL, cookies string) { - glog.Infof("disk storage run SetCookies, %v, %s", u, cookies) + //glog.Infof("disk storage run SetCookies, %v, %s", u, cookies) filePath := fmt.Sprintf("%s"+ds.fileName, u.Hostname()) if err := os.WriteFile(filePath, []byte(cookies), 0644); err != nil { log.Fatal(err)