support cookies by account

.gitignore

@@ -16,6 +16,7 @@
 build/
 /warmane
 dev_config.yml
+*.cookies
 
 .ropeproject/
 vendor/

README.md

@@ -8,6 +8,7 @@
 + [x] 验证码识别
 + [x] 自动Points签到
 + [x] 多账户支持
++ [x] 支持cookies保存,免登录
 + [ ] 商城人物数据爬取
 + [ ] 人物数据分析并存储
 + [ ] 多维度查询

engine/engine.go

@@ -2,6 +2,7 @@ package engine
 
 import (
 	"encoding/json"
+	"fmt"
 	"github.com/gocolly/colly/v2"
 	"github.com/golang/glog"
 	"gitub.com/zJiajun/warmane/internal/captcha"
@@ -31,7 +32,7 @@ const (
 
 type Engine struct {
 	config    *config.Config
-	scraper   *scraper.Scraper
+	scrapers  map[string]*scraper.Scraper
 	wg        sync.WaitGroup
 	csrfToken string
 }
@@ -42,8 +43,8 @@ func New() *Engine {
 		panic(err)
 	}
 	return &Engine{
-		config:  conf,
-		scraper: scraper.NewScraper(),
+		config:   conf,
+		scrapers: make(map[string]*scraper.Scraper, len(conf.Accounts)),
 	}
 }
 
@@ -55,11 +56,20 @@ func (e *Engine) RunDailyPoints() {
 	e.wg.Add(count)
 	glog.Infof("开始goroutine并发处理")
 	for _, account := range e.config.Accounts {
+		e.setScraper(account)
 		go e.loginAndCollect(account)
 	}
 	e.wg.Wait()
 }
 
+func (e *Engine) setScraper(account config.Account) {
+	e.scrapers[account.Username] = scraper.NewScraper(account.Username)
+}
+
+func (e *Engine) scraper(account config.Account) *scraper.Scraper {
+	return e.scrapers[account.Username]
+}
+
 func (e *Engine) loginAndCollect(account config.Account) {
 	defer e.wg.Done()
 	if err := e.login(account); err != nil {
@@ -79,11 +89,11 @@ func (e *Engine) loginAndCollect(account config.Account) {
 }
 
 func (e *Engine) login(account config.Account) error {
-	if e.existCookiesFile() {
-		glog.Infof("存在cookies文件,跳过登录")
+	if e.existCookiesFile(account) {
+		glog.Infof("存在[%s]cookies文件,跳过登录", account.Username)
 		return nil
 	}
-	c := e.scraper.CloneCollector()
+	c := e.scraper(account).CloneCollector()
 	var err error
 	e.csrfToken, err = getCsrfToken(c)
 	if err != nil {
@@ -102,8 +112,8 @@ func (e *Engine) login(account config.Account) error {
 		"g-recaptcha-response": code,
 		"userRM":               "on",
 	}
-	e.scraper.SetRequestHeaders(c, e.csrfToken)
-	e.scraper.DecodeResponse(c)
+	e.scraper(account).SetRequestHeaders(c, e.csrfToken)
+	e.scraper(account).DecodeResponse(c)
 	var bodyMsg BodyMsg
 	c.OnResponse(func(response *colly.Response) {
 		bodyText := string(response.Body)
@@ -127,25 +137,26 @@ func (e *Engine) login(account config.Account) error {
 	return err
 }
 
-func (e *Engine) existCookiesFile() bool {
-	_, err := os.Stat("www.warmane.com.cookies")
+func (e *Engine) existCookiesFile(account config.Account) bool {
+	cookiesFile := fmt.Sprintf("www.warmane.com.%s.cookies", account.Username)
+	_, err := os.Stat(cookiesFile)
 	if os.IsNotExist(err) {
 		return false
 	}
 	return true
 }
 
 func (e *Engine) collectPoints(account config.Account) error {
-	beforeCoins, beforePoints, err := e.getInfo()
+	beforeCoins, beforePoints, err := e.getInfo(account)
 	if err != nil {
 		return err
 	}
 	glog.Infof("账号[%s]收集签到点[前]的 coins: [%s], points: [%s]",
 		account.Username, beforeCoins, beforePoints)
 
-	c := e.scraper.CloneCollector()
-	e.scraper.SetRequestHeaders(c, e.csrfToken)
-	e.scraper.DecodeResponse(c)
+	c := e.scraper(account).CloneCollector()
+	e.scraper(account).SetRequestHeaders(c, e.csrfToken)
+	e.scraper(account).DecodeResponse(c)
 	var bodyMsg BodyMsg
 	c.OnResponse(func(response *colly.Response) {
 		bodyText := string(response.Body)
@@ -170,7 +181,7 @@ func (e *Engine) collectPoints(account config.Account) error {
 	if err != nil {
 		return err
 	}
-	afterCoins, afterPoints, err := e.getInfo()
+	afterCoins, afterPoints, err := e.getInfo(account)
 	if err != nil {
 		return err
 	}
@@ -180,7 +191,7 @@ func (e *Engine) collectPoints(account config.Account) error {
 }
 
 func (e *Engine) logout(account config.Account) error {
-	c := e.scraper.CloneCollector()
+	c := e.scraper(account).CloneCollector()
 	err := c.Visit(config.LogoutUrl)
 	if err != nil {
 		return err
@@ -189,10 +200,10 @@ func (e *Engine) logout(account config.Account) error {
 	return err
 }
 
-func (e *Engine) getInfo() (coins string, points string, err error) {
-	c := e.scraper.CloneCollector()
-	e.scraper.SetRequestHeaders(c, e.csrfToken)
-	e.scraper.DecodeResponse(c)
+func (e *Engine) getInfo(account config.Account) (coins string, points string, err error) {
+	c := e.scraper(account).CloneCollector()
+	e.scraper(account).SetRequestHeaders(c, e.csrfToken)
+	e.scraper(account).DecodeResponse(c)
 	c.OnHTML(config.CoinsSelector, func(element *colly.HTMLElement) {
 		coins = element.Text
 	})

internal/scraper/scraper.go

@@ -13,7 +13,7 @@ type Scraper struct {
 	c *colly.Collector
 }
 
-func NewScraper() *Scraper {
+func NewScraper(name string) *Scraper {
 	s := &Scraper{
 		c: colly.NewCollector(
 			colly.AllowURLRevisit(),
@@ -22,7 +22,7 @@ func NewScraper() *Scraper {
 		),
 	}
 	s.c.SetRequestTimeout(600 * time.Second)
-	if err := s.c.SetStorage(storage.NewDiskStorage()); err != nil {
+	if err := s.c.SetStorage(storage.NewDiskStorage(name)); err != nil {
 		panic(err)
 	}
 	return s
@@ -45,7 +45,7 @@ func (s *Scraper) SetRequestHeaders(c *colly.Collector, csrfToken string) {
 
 func (s *Scraper) DecodeResponse(c *colly.Collector) {
 	c.OnResponse(func(response *colly.Response) {
-		glog.Infof("onResponse [%s], statusCode:[%d], size:[%d]", response.Request.URL, response.StatusCode, len(response.Body))
+		//glog.Infof("onResponse [%s], statusCode:[%d], size:[%d]", response.Request.URL, response.StatusCode, len(response.Body))
 		encoding := response.Headers.Get("Content-Encoding")
 		if encoding == "" {
 			return
@@ -56,7 +56,7 @@ func (s *Scraper) DecodeResponse(c *colly.Collector) {
 			return
 		}
 		response.Body = decodeResp
-		glog.Infof("onResponse decode [%s] response success", response.Request.URL)
+		//glog.Infof("onResponse decode [%s] response success", response.Request.URL)
 	})
 }
 

internal/storage/diskstorage.go

@@ -3,7 +3,6 @@ package storage
 import (
 	"fmt"
 	"github.com/gocolly/colly/v2/storage"
-	"github.com/golang/glog"
 	"log"
 	"net/url"
 	"os"
@@ -14,9 +13,9 @@ type DiskStorage struct {
 	inMemoryStorage *storage.InMemoryStorage
 }
 
-func NewDiskStorage() *DiskStorage {
+func NewDiskStorage(name string) *DiskStorage {
 	return &DiskStorage{
-		fileName:        ".cookies",
+		fileName:        "." + name + ".cookies",
 		inMemoryStorage: &storage.InMemoryStorage{},
 	}
 }
@@ -34,7 +33,7 @@ func (ds *DiskStorage) IsVisited(requestID uint64) (bool, error) {
 }
 
 func (ds *DiskStorage) Cookies(u *url.URL) string {
-	glog.Infof("disk storage run cookies, %v", u)
+	//glog.Infof("disk storage run cookies, %v", u)
 	filePath := fmt.Sprintf("%s"+ds.fileName, u.Hostname())
 	buf, err := os.ReadFile(filePath)
 	if err != nil {
@@ -44,7 +43,7 @@ func (ds *DiskStorage) Cookies(u *url.URL) string {
 }
 
 func (ds *DiskStorage) SetCookies(u *url.URL, cookies string) {
-	glog.Infof("disk storage run SetCookies, %v, %s", u, cookies)
+	//glog.Infof("disk storage run SetCookies, %v, %s", u, cookies)
 	filePath := fmt.Sprintf("%s"+ds.fileName, u.Hostname())
 	if err := os.WriteFile(filePath, []byte(cookies), 0644); err != nil {
 		log.Fatal(err)