From 65de499edb7174d785719a604b5708fa6f5d181e Mon Sep 17 00:00:00 2001 From: zjiajun Date: Wed, 28 Aug 2024 11:02:19 +0800 Subject: [PATCH] refactor --- .goreleaser.yml | 2 -- config/config.go | 32 -------------------------------- constant/const.go | 2 +- engine/account.go | 29 +++++++++++++++++++++++++---- engine/engine.go | 6 +----- engine/login.go | 16 +++++++--------- engine/points.go | 2 -- engine/trade.go | 24 ++++-------------------- errors/errors.go | 29 ----------------------------- model/table/visited.go | 15 --------------- scraper/scraper.go | 14 +++++--------- scraper/storage/sqlitestorage.go | 8 +------- 12 files changed, 44 insertions(+), 135 deletions(-) delete mode 100644 config/config.go delete mode 100644 errors/errors.go delete mode 100644 model/table/visited.go diff --git a/.goreleaser.yml b/.goreleaser.yml index d7cee8a..b7d6c43 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -38,8 +38,6 @@ archives: wrap_in_directory: false files: - README*.md - - config.yml - checksum: name_template: '{{ .ProjectName }}_checksums.txt' diff --git a/config/config.go b/config/config.go deleted file mode 100644 index 59742ff..0000000 --- a/config/config.go +++ /dev/null @@ -1,32 +0,0 @@ -package config - -import ( - "github.com/zJiajun/warmane/errors" - "gopkg.in/yaml.v3" - "os" -) - -type Config struct { - CaptchaApiKey string `yaml:"captchaApiKey"` - WarmaneSiteKey string `yaml:"warmaneSiteKey"` - UseCookiesLogin bool `yaml:"useCookiesLogin"` - Accounts []*Account `yaml:"accounts"` -} - -type Account struct { - Username string `yaml:"username"` - Password string `yaml:"password"` -} - -func Load(cfg string) (*Config, error) { - file, err := os.ReadFile(cfg) - if err != nil { - return nil, errors.ErrConfNotFound - } - var conf Config - err = yaml.Unmarshal(file, &conf) - if err != nil { - return nil, errors.ErrConfDecodeError - } - return &conf, nil -} diff --git a/constant/const.go b/constant/const.go index 092270b..aca85f5 100644 --- a/constant/const.go +++ b/constant/const.go @@ -21,7 +21,7 @@ const ( ) const WarmaneSiteKey = "6LfXRRsUAAAAAEApnVwrtQ7aFprn4naEcc05AZUR" -const captchaApiKey = "" +const CaptchaApiKey = "" const ( ONLINE_STATUS = "online" diff --git a/engine/account.go b/engine/account.go index 97b3aac..dba2c20 100644 --- a/engine/account.go +++ b/engine/account.go @@ -2,12 +2,15 @@ package engine import ( "errors" + "fmt" "github.com/gocolly/colly/v2" "github.com/gocolly/colly/v2/storage" "github.com/zJiajun/warmane/constant" "github.com/zJiajun/warmane/logger" "github.com/zJiajun/warmane/model/table" "math/rand/v2" + "net/url" + "strconv" "strings" "sync" "time" @@ -111,8 +114,10 @@ func (e *Engine) validateCookiesKey(cookies string) error { func (e *Engine) validAndFetchAccountInfo(accountName string, accountDetails *table.AccountDetails) error { isLogin, _ := false, false c := e.getScraper(accountName).CloneCollector() - e.getScraper(accountName).SetRequestHeaders(c) - e.getScraper(accountName).DecodeResponse(c) + + c.OnResponse(func(response *colly.Response) { + logger.Info((string)(response.Body)) + }) c.OnHTML("div.content-inner.left > table > tbody > tr:nth-child(2) > td", func(element *colly.HTMLElement) { isLogin = strings.Contains(element.Text, accountName) }) @@ -126,8 +131,8 @@ func (e *Engine) validAndFetchAccountInfo(accountName string, accountDetails *ta c.OnHTML(".myPoints", func(element *colly.HTMLElement) { accountDetails.Points = element.Text }) - c.OnHTML("div.content-inner.left > table > tbody > tr:nth-child(6) > td", func(element *colly.HTMLElement) { - accountDetails.Email = strings.TrimSpace(strings.Split(element.Text, ":")[1]) + c.OnHTML("div.content-inner.left > table > tbody > tr:nth-child(6) > td > a", func(element *colly.HTMLElement) { + accountDetails.Email = strings.TrimSpace(decodeEmail(element.Attr("data-cfemail"))) }) c.OnHTML("div.content-inner.right > table > tbody > tr:nth-child(2) > td", func(element *colly.HTMLElement) { accountDetails.Status = strings.TrimSpace(strings.Split(element.Text, ":")[1]) @@ -157,6 +162,22 @@ func (e *Engine) validAndFetchAccountInfo(accountName string, accountDetails *ta return nil } +func decodeEmail(encodedEmail string) string { + if encodedEmail == "" { + return "[email protected]" + } + r, _ := strconv.ParseInt(encodedEmail[0:2], 16, 0) + n := 2 + decoded := "" + for n < len(encodedEmail) { + part, _ := strconv.ParseInt(encodedEmail[n:n+2], 16, 0) + decoded += "%" + fmt.Sprintf("%0.2x", int(part)^int(r)) + n += 2 + } + unquoted, _ := url.QueryUnescape(decoded) + return unquoted +} + func (e *Engine) updateAccountInfo(account *table.Account, accountDetails *table.AccountDetails) error { result := e.db.Model(&table.Account{}).Where("id = ?", account.ID).Where("status = ?", constant.OFFLINE_STATUS).Update("status", constant.ONLINE_STATUS) if result.Error != nil { diff --git a/engine/engine.go b/engine/engine.go index dca0b1a..af7858f 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -1,7 +1,6 @@ package engine import ( - "github.com/zJiajun/warmane/config" "github.com/zJiajun/warmane/database" "github.com/zJiajun/warmane/logger" "github.com/zJiajun/warmane/model/table" @@ -14,8 +13,6 @@ type Engine struct { scrapers *scraper.Scrapers db *gorm.DB wg sync.WaitGroup - //TODO: will be deleted - config *config.Config } func New() *Engine { @@ -42,7 +39,7 @@ func (e *Engine) init() { } for _, account := range accounts { e.scrapers.GetOrPut(account.AccountName) - go e.keepingAccountOnline(int64(account.ID)) + //go e.keepingAccountOnline(int64(account.ID)) } } @@ -55,6 +52,5 @@ func autoMigrate(db *gorm.DB) error { &table.Account{}, &table.AccountDetails{}, &table.TradeInfo{}, - &table.Visited{}, ) } diff --git a/engine/login.go b/engine/login.go index af44119..536b1f4 100644 --- a/engine/login.go +++ b/engine/login.go @@ -5,16 +5,16 @@ import ( "fmt" "github.com/gocolly/colly/v2" "github.com/zJiajun/warmane/captcha" - "github.com/zJiajun/warmane/config" "github.com/zJiajun/warmane/constant" "github.com/zJiajun/warmane/logger" + "github.com/zJiajun/warmane/model/table" "github.com/zJiajun/warmane/scraper/storage" ) -func (e *Engine) login(account *config.Account) error { - name := account.Username +func (e *Engine) captchaLogin(account *table.Account) error { + name := account.AccountName logger.Infof("配置项[useCookiesLogin]为false, 使用2captcha方式登录") - capt := captcha.NewCaptcha(e.config.CaptchaApiKey, e.config.WarmaneSiteKey, constant.LoginUrl) + capt := captcha.NewCaptcha(constant.CaptchaApiKey, constant.WarmaneSiteKey, constant.LoginUrl) loginData := make(map[string]string, 5) if code, err := capt.HandleCaptcha(); err == nil { loginData["return"] = "" @@ -27,8 +27,6 @@ func (e *Engine) login(account *config.Account) error { } _ = storage.Clear(e.db, name) c := e.getScraper(name).CloneCollector() - e.getScraper(name).SetRequestHeaders(c) - e.getScraper(name).DecodeResponse(c) var bodyErr error var bodyMsg struct { Messages struct { @@ -58,12 +56,12 @@ func (e *Engine) login(account *config.Account) error { return nil } -func (e *Engine) logout(account *config.Account) error { - c := e.getScraper(account.Username).CloneCollector() +func (e *Engine) logout(accountName string) error { + c := e.getScraper(accountName).CloneCollector() err := c.Visit(constant.LogoutUrl) if err != nil { return err } - logger.Infof("账号[%s]退出成功", account.Username) + logger.Infof("账号[%s]退出成功", accountName) return err } diff --git a/engine/points.go b/engine/points.go index 686ac46..26c526c 100644 --- a/engine/points.go +++ b/engine/points.go @@ -44,8 +44,6 @@ func (e *Engine) CollectAccountPoints(accountId int64) error { e.db.First(&account, accountId) name := account.AccountName c := e.getScraper(name).CloneCollector() - e.getScraper(name).SetRequestHeaders(c) - e.getScraper(name).DecodeResponse(c) var bodyMsg struct { Messages struct { Success []string `json:"success"` diff --git a/engine/trade.go b/engine/trade.go index 2c3a53c..e32ccf9 100644 --- a/engine/trade.go +++ b/engine/trade.go @@ -5,7 +5,6 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/gocolly/colly/v2" "github.com/zJiajun/warmane/common" - "github.com/zJiajun/warmane/config" "github.com/zJiajun/warmane/constant" "github.com/zJiajun/warmane/logger" "github.com/zJiajun/warmane/model" @@ -23,36 +22,21 @@ var ( onyxiaRealm = &common.Pair[string, string]{Left: "14", Right: "Onyxia"} ) -func (e *Engine) RunTradeData() { - logger.Info("开始运行商场角色交易数据爬取") - account := e.config.Accounts[0] - if err := e.login(account); err != nil { - logger.Errorf("账号[%s]登录错误, 原因: %v", account.Username, err) - return - } - if err := e.trade(account); err != nil { - logger.Errorf("账号[%s]查询商场数据错误, 原因: %v", account.Username, err) - return - } -} - -func (e *Engine) trade(account *config.Account) error { +func (e *Engine) trade(account *table.Account) error { trades, err := e.fetchTradeData(account) if err != nil { return err } - err = e.storeTradeData(account.Username, trades) + err = e.storeTradeData(account.AccountName, trades) if err != nil { return err } return nil } -func (e *Engine) fetchTradeData(account *config.Account) ([]*table.TradeInfo, error) { - name := account.Username +func (e *Engine) fetchTradeData(account *table.Account) ([]*table.TradeInfo, error) { + name := account.AccountName c := e.getScraper(name).CloneCollector() - e.getScraper(name).SetRequestHeaders(c) - e.getScraper(name).DecodeResponse(c) var tradeResp struct { Content []string `json:"content"` } diff --git a/errors/errors.go b/errors/errors.go deleted file mode 100644 index 99003f2..0000000 --- a/errors/errors.go +++ /dev/null @@ -1,29 +0,0 @@ -package errors - -import ( - "errors" - "github.com/zJiajun/warmane/logger" -) - -var ( - ErrConfNotFound = errors.New("配置文件[config.yml]未找到, 请把配置文件放到程序同一目录下") - ErrConfDecodeError = errors.New("配置文件[config.yml]解析错误, 请检查配置文件") - ErrCookieNotFound = errors.New("cookies数据未找到") - ErrCookieNonMatchKey = errors.New("cookies数据存在,但不匹配cookiesKey") -) - -func HandleError(err error) { - if err == nil { - return - } - switch { - case errors.Is(err, ErrConfNotFound): - logger.Error(err.Error()) - case errors.Is(err, ErrConfDecodeError): - logger.Error(err.Error()) - case errors.Is(err, ErrConfNotFound): - logger.Error(err.Error()) - default: - logger.Error(err.Error()) - } -} diff --git a/model/table/visited.go b/model/table/visited.go deleted file mode 100644 index 828b7c3..0000000 --- a/model/table/visited.go +++ /dev/null @@ -1,15 +0,0 @@ -package table - -import "gorm.io/gorm" - -const VisitedTableName = "visited" - -type Visited struct { - RequestID int `gorm:"index"` - Visited int - gorm.Model -} - -func (*Visited) TableName() string { - return VisitedTableName -} diff --git a/scraper/scraper.go b/scraper/scraper.go index acef060..61af1d8 100644 --- a/scraper/scraper.go +++ b/scraper/scraper.go @@ -2,9 +2,6 @@ package scraper import ( "github.com/gocolly/colly/v2" - "github.com/zJiajun/warmane/constant" - "github.com/zJiajun/warmane/logger" - "github.com/zJiajun/warmane/scraper/internal/decode" "github.com/zJiajun/warmane/scraper/internal/extensions" "github.com/zJiajun/warmane/scraper/storage" "gorm.io/gorm" @@ -34,20 +31,16 @@ func newScraper(name string, db *gorm.DB) *Scraper { }); err != nil { panic(err) } - /* - if err := s.c.SetStorage(storage.NewDiskStorage(name)); err != nil { - panic(err) - } - */ if err := s.c.SetStorage(storage.NewSqliteStorage(name, db)); err != nil { panic(err) } return s } +/* func (s *Scraper) SetRequestHeaders(c *colly.Collector) { c.OnRequest(func(request *colly.Request) { - request.Headers.Set("Accept", "application/json, text/javascript, */*; q=0.01") + request.Headers.Set("Accept", "application/json, text/javascript") request.Headers.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7") request.Headers.Set("Accept-Encoding", "gzip, deflate, br") request.Headers.Set("Cache-Control", "no-cache") @@ -67,7 +60,9 @@ func (s *Scraper) SetRequestHeaders(c *colly.Collector) { request.Headers.Set("X-Requested-With", "XMLHttpRequest") }) } +*/ +/* func (s *Scraper) DecodeResponse(c *colly.Collector) { c.OnResponse(func(response *colly.Response) { encoding := response.Headers.Get("Content-Encoding") @@ -82,6 +77,7 @@ func (s *Scraper) DecodeResponse(c *colly.Collector) { response.Body = decodeResp }) } +*/ func (s *Scraper) CloneCollector() *colly.Collector { return s.c.Clone() diff --git a/scraper/storage/sqlitestorage.go b/scraper/storage/sqlitestorage.go index efb92f7..3b9c25d 100644 --- a/scraper/storage/sqlitestorage.go +++ b/scraper/storage/sqlitestorage.go @@ -24,16 +24,10 @@ func (s *SqliteStorage) Init() error { } func (s *SqliteStorage) Visited(requestID uint64) error { - visited := &table.Visited{RequestID: int(requestID), Visited: 1} - return s.db.Create(visited).Error + return nil } func (s *SqliteStorage) IsVisited(requestID uint64) (bool, error) { - var count int64 - s.db.Model(&table.Visited{}).Where("request_id = ?", requestID).Count(&count) - if count >= 1 { - return true, nil - } return false, nil }