refactor

.goreleaser.yml

@@ -38,8 +38,6 @@ archives:
     wrap_in_directory: false
     files:
       - README*.md
-      - config.yml
-
 
 checksum:
   name_template: '{{ .ProjectName }}_checksums.txt'

constant/const.go

@@ -21,7 +21,7 @@ const (
 )
 
 const WarmaneSiteKey = "6LfXRRsUAAAAAEApnVwrtQ7aFprn4naEcc05AZUR"
-const captchaApiKey = ""
+const CaptchaApiKey = ""
 
 const (
 	ONLINE_STATUS  = "online"

engine/account.go

@@ -2,12 +2,15 @@ package engine
 
 import (
 	"errors"
+	"fmt"
 	"github.com/gocolly/colly/v2"
 	"github.com/gocolly/colly/v2/storage"
 	"github.com/zJiajun/warmane/constant"
 	"github.com/zJiajun/warmane/logger"
 	"github.com/zJiajun/warmane/model/table"
 	"math/rand/v2"
+	"net/url"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -111,8 +114,10 @@ func (e *Engine) validateCookiesKey(cookies string) error {
 func (e *Engine) validAndFetchAccountInfo(accountName string, accountDetails *table.AccountDetails) error {
 	isLogin, _ := false, false
 	c := e.getScraper(accountName).CloneCollector()
-	e.getScraper(accountName).SetRequestHeaders(c)
-	e.getScraper(accountName).DecodeResponse(c)
+
+	c.OnResponse(func(response *colly.Response) {
+		logger.Info((string)(response.Body))
+	})
 	c.OnHTML("div.content-inner.left > table > tbody > tr:nth-child(2) > td", func(element *colly.HTMLElement) {
 		isLogin = strings.Contains(element.Text, accountName)
 	})
@@ -126,8 +131,8 @@ func (e *Engine) validAndFetchAccountInfo(accountName string, accountDetails *ta
 	c.OnHTML(".myPoints", func(element *colly.HTMLElement) {
 		accountDetails.Points = element.Text
 	})
-	c.OnHTML("div.content-inner.left > table > tbody > tr:nth-child(6) > td", func(element *colly.HTMLElement) {
-		accountDetails.Email = strings.TrimSpace(strings.Split(element.Text, ":")[1])
+	c.OnHTML("div.content-inner.left > table > tbody > tr:nth-child(6) > td > a", func(element *colly.HTMLElement) {
+		accountDetails.Email = strings.TrimSpace(decodeEmail(element.Attr("data-cfemail")))
 	})
 	c.OnHTML("div.content-inner.right > table > tbody > tr:nth-child(2) > td", func(element *colly.HTMLElement) {
 		accountDetails.Status = strings.TrimSpace(strings.Split(element.Text, ":")[1])
@@ -157,6 +162,22 @@ func (e *Engine) validAndFetchAccountInfo(accountName string, accountDetails *ta
 	return nil
 }
 
+func decodeEmail(encodedEmail string) string {
+	if encodedEmail == "" {
+		return "[email protected]"
+	}
+	r, _ := strconv.ParseInt(encodedEmail[0:2], 16, 0)
+	n := 2
+	decoded := ""
+	for n < len(encodedEmail) {
+		part, _ := strconv.ParseInt(encodedEmail[n:n+2], 16, 0)
+		decoded += "%" + fmt.Sprintf("%0.2x", int(part)^int(r))
+		n += 2
+	}
+	unquoted, _ := url.QueryUnescape(decoded)
+	return unquoted
+}
+
 func (e *Engine) updateAccountInfo(account *table.Account, accountDetails *table.AccountDetails) error {
 	result := e.db.Model(&table.Account{}).Where("id = ?", account.ID).Where("status = ?", constant.OFFLINE_STATUS).Update("status", constant.ONLINE_STATUS)
 	if result.Error != nil {

engine/engine.go

@@ -1,7 +1,6 @@
 package engine
 
 import (
-	"github.com/zJiajun/warmane/config"
 	"github.com/zJiajun/warmane/database"
 	"github.com/zJiajun/warmane/logger"
 	"github.com/zJiajun/warmane/model/table"
@@ -14,8 +13,6 @@ type Engine struct {
 	scrapers *scraper.Scrapers
 	db       *gorm.DB
 	wg       sync.WaitGroup
-	//TODO: will be deleted
-	config *config.Config
 }
 
 func New() *Engine {
@@ -42,7 +39,7 @@ func (e *Engine) init() {
 	}
 	for _, account := range accounts {
 		e.scrapers.GetOrPut(account.AccountName)
-		go e.keepingAccountOnline(int64(account.ID))
+		//go e.keepingAccountOnline(int64(account.ID))
 	}
 }
 
@@ -55,6 +52,5 @@ func autoMigrate(db *gorm.DB) error {
 		&table.Account{},
 		&table.AccountDetails{},
 		&table.TradeInfo{},
-		&table.Visited{},
 	)
 }

engine/login.go

@@ -5,16 +5,16 @@ import (
 	"fmt"
 	"github.com/gocolly/colly/v2"
 	"github.com/zJiajun/warmane/captcha"
-	"github.com/zJiajun/warmane/config"
 	"github.com/zJiajun/warmane/constant"
 	"github.com/zJiajun/warmane/logger"
+	"github.com/zJiajun/warmane/model/table"
 	"github.com/zJiajun/warmane/scraper/storage"
 )
 
-func (e *Engine) login(account *config.Account) error {
-	name := account.Username
+func (e *Engine) captchaLogin(account *table.Account) error {
+	name := account.AccountName
 	logger.Infof("配置项[useCookiesLogin]为false, 使用2captcha方式登录")
-	capt := captcha.NewCaptcha(e.config.CaptchaApiKey, e.config.WarmaneSiteKey, constant.LoginUrl)
+	capt := captcha.NewCaptcha(constant.CaptchaApiKey, constant.WarmaneSiteKey, constant.LoginUrl)
 	loginData := make(map[string]string, 5)
 	if code, err := capt.HandleCaptcha(); err == nil {
 		loginData["return"] = ""
@@ -27,8 +27,6 @@ func (e *Engine) login(account *config.Account) error {
 	}
 	_ = storage.Clear(e.db, name)
 	c := e.getScraper(name).CloneCollector()
-	e.getScraper(name).SetRequestHeaders(c)
-	e.getScraper(name).DecodeResponse(c)
 	var bodyErr error
 	var bodyMsg struct {
 		Messages struct {
@@ -58,12 +56,12 @@ func (e *Engine) login(account *config.Account) error {
 	return nil
 }
 
-func (e *Engine) logout(account *config.Account) error {
-	c := e.getScraper(account.Username).CloneCollector()
+func (e *Engine) logout(accountName string) error {
+	c := e.getScraper(accountName).CloneCollector()
 	err := c.Visit(constant.LogoutUrl)
 	if err != nil {
 		return err
 	}
-	logger.Infof("账号[%s]退出成功", account.Username)
+	logger.Infof("账号[%s]退出成功", accountName)
 	return err
 }

engine/points.go

@@ -44,8 +44,6 @@ func (e *Engine) CollectAccountPoints(accountId int64) error {
 	e.db.First(&account, accountId)
 	name := account.AccountName
 	c := e.getScraper(name).CloneCollector()
-	e.getScraper(name).SetRequestHeaders(c)
-	e.getScraper(name).DecodeResponse(c)
 	var bodyMsg struct {
 		Messages struct {
 			Success []string `json:"success"`

engine/trade.go

@@ -5,7 +5,6 @@ import (
 	"github.com/PuerkitoBio/goquery"
 	"github.com/gocolly/colly/v2"
 	"github.com/zJiajun/warmane/common"
-	"github.com/zJiajun/warmane/config"
 	"github.com/zJiajun/warmane/constant"
 	"github.com/zJiajun/warmane/logger"
 	"github.com/zJiajun/warmane/model"
@@ -23,36 +22,21 @@ var (
 	onyxiaRealm    = &common.Pair[string, string]{Left: "14", Right: "Onyxia"}
 )
 
-func (e *Engine) RunTradeData() {
-	logger.Info("开始运行商场角色交易数据爬取")
-	account := e.config.Accounts[0]
-	if err := e.login(account); err != nil {
-		logger.Errorf("账号[%s]登录错误, 原因: %v", account.Username, err)
-		return
-	}
-	if err := e.trade(account); err != nil {
-		logger.Errorf("账号[%s]查询商场数据错误, 原因: %v", account.Username, err)
-		return
-	}
-}
-
-func (e *Engine) trade(account *config.Account) error {
+func (e *Engine) trade(account *table.Account) error {
 	trades, err := e.fetchTradeData(account)
 	if err != nil {
 		return err
 	}
-	err = e.storeTradeData(account.Username, trades)
+	err = e.storeTradeData(account.AccountName, trades)
 	if err != nil {
 		return err
 	}
 	return nil
 }
 
-func (e *Engine) fetchTradeData(account *config.Account) ([]*table.TradeInfo, error) {
-	name := account.Username
+func (e *Engine) fetchTradeData(account *table.Account) ([]*table.TradeInfo, error) {
+	name := account.AccountName
 	c := e.getScraper(name).CloneCollector()
-	e.getScraper(name).SetRequestHeaders(c)
-	e.getScraper(name).DecodeResponse(c)
 	var tradeResp struct {
 		Content []string `json:"content"`
 	}

scraper/scraper.go

@@ -2,9 +2,6 @@ package scraper
 
 import (
 	"github.com/gocolly/colly/v2"
-	"github.com/zJiajun/warmane/constant"
-	"github.com/zJiajun/warmane/logger"
-	"github.com/zJiajun/warmane/scraper/internal/decode"
 	"github.com/zJiajun/warmane/scraper/internal/extensions"
 	"github.com/zJiajun/warmane/scraper/storage"
 	"gorm.io/gorm"
@@ -34,20 +31,16 @@ func newScraper(name string, db *gorm.DB) *Scraper {
 	}); err != nil {
 		panic(err)
 	}
-	/*
-		if err := s.c.SetStorage(storage.NewDiskStorage(name)); err != nil {
-			panic(err)
-		}
-	*/
 	if err := s.c.SetStorage(storage.NewSqliteStorage(name, db)); err != nil {
 		panic(err)
 	}
 	return s
 }
 
+/*
 func (s *Scraper) SetRequestHeaders(c *colly.Collector) {
 	c.OnRequest(func(request *colly.Request) {
-		request.Headers.Set("Accept", "application/json, text/javascript, */*; q=0.01")
+		request.Headers.Set("Accept", "application/json, text/javascript")
 		request.Headers.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7")
 		request.Headers.Set("Accept-Encoding", "gzip, deflate, br")
 		request.Headers.Set("Cache-Control", "no-cache")
@@ -67,7 +60,9 @@ func (s *Scraper) SetRequestHeaders(c *colly.Collector) {
 		request.Headers.Set("X-Requested-With", "XMLHttpRequest")
 	})
 }
+*/
 
+/*
 func (s *Scraper) DecodeResponse(c *colly.Collector) {
 	c.OnResponse(func(response *colly.Response) {
 		encoding := response.Headers.Get("Content-Encoding")
@@ -82,6 +77,7 @@ func (s *Scraper) DecodeResponse(c *colly.Collector) {
 		response.Body = decodeResp
 	})
 }
+*/
 
 func (s *Scraper) CloneCollector() *colly.Collector {
 	return s.c.Clone()

scraper/storage/sqlitestorage.go

@@ -24,16 +24,10 @@ func (s *SqliteStorage) Init() error {
 }
 
 func (s *SqliteStorage) Visited(requestID uint64) error {
-	visited := &table.Visited{RequestID: int(requestID), Visited: 1}
-	return s.db.Create(visited).Error
+	return nil
 }
 
 func (s *SqliteStorage) IsVisited(requestID uint64) (bool, error) {
-	var count int64
-	s.db.Model(&table.Visited{}).Where("request_id = ?", requestID).Count(&count)
-	if count >= 1 {
-		return true, nil
-	}
 	return false, nil
 }