fix: modify CacheEmptyRooms

west2-online · Sep 29, 2024 · 076d6f1 · 076d6f1
1 parent ef1fe44
commit 076d6f1
Show file tree

Hide file tree

Showing 20 changed files with 135 additions and 49 deletions.
diff --git a/cmd/api/biz/handler/api/class_room_service.go b/cmd/api/biz/handler/api/class_room_service.go
diff --git a/cmd/api/biz/handler/api/user_service.go b/cmd/api/biz/handler/api/user_service.go
diff --git a/cmd/api/biz/rpc/classroom.go b/cmd/api/biz/rpc/classroom.go
@@ -9,6 +9,7 @@ import (
 	"github.com/west2-online/fzuhelper-server/kitex_gen/classroom/classroomservice"
 	"github.com/west2-online/fzuhelper-server/kitex_gen/model"
 	"github.com/west2-online/fzuhelper-server/pkg/constants"
+	"github.com/west2-online/fzuhelper-server/pkg/logger"
 	"github.com/west2-online/fzuhelper-server/pkg/utils"
 )
 
@@ -21,13 +22,13 @@ func InitClassroomRPC() {
 	if err != nil {
 		panic(err)
 	}
-	utils.LoggerObj.Info("InitClassroomRPC success")
+	logger.LoggerObj.Info("InitClassroomRPC success")
 }
 
 func GetEmptyRoomRPC(ctx context.Context, req *classroom.EmptyRoomRequest) (emptyRooms []*model.Classroom, err error) {
 	resp, err := classroomClient.GetEmptyRoom(ctx, req)
 	if err != nil {
-		utils.LoggerObj.Errorf("api.rpc.classroom GetEmptyRoomRPC received rpc error %v", err)
+		logger.LoggerObj.Errorf("api.rpc.classroom GetEmptyRoomRPC received rpc error %v", err)
 		return nil, err
 	}
 	if err = utils.IsSuccess(resp.Base); err != nil {

diff --git a/cmd/api/biz/rpc/user.go b/cmd/api/biz/rpc/user.go
@@ -8,6 +8,7 @@ import (
 	"github.com/west2-online/fzuhelper-server/kitex_gen/user"
 	"github.com/west2-online/fzuhelper-server/kitex_gen/user/userservice"
 	"github.com/west2-online/fzuhelper-server/pkg/constants"
+	"github.com/west2-online/fzuhelper-server/pkg/logger"
 	"github.com/west2-online/fzuhelper-server/pkg/utils"
 )
 
@@ -20,13 +21,13 @@ func InitUserRPC() {
 	if err != nil {
 		panic(err)
 	}
-	utils.LoggerObj.Info("InitUserRPC success")
+	logger.LoggerObj.Info("InitUserRPC success")
 }
 
 func GetLoginDataRPC(ctx context.Context, req *user.GetLoginDataRequest) (string, []string, error) {
 	resp, err := userClient.GetLoginData(ctx, req)
 	if err != nil {
-		utils.LoggerObj.Errorf("api.rpc.user GetLoginDataRPC received rpc error %v", err)
+		logger.LoggerObj.Errorf("api.rpc.user GetLoginDataRPC received rpc error %v", err)
 		return "", nil, err
 	}
 	if err = utils.IsSuccess(resp.Base); err != nil {

diff --git a/cmd/api/main.go b/cmd/api/main.go
diff --git a/cmd/classroom/dal/cache/empty_room.go b/cmd/classroom/dal/cache/empty_room.go
@@ -4,7 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"github.com/west2-online/fzuhelper-server/pkg/constants"
-	"github.com/west2-online/fzuhelper-server/pkg/utils"
+	"github.com/west2-online/fzuhelper-server/pkg/logger"
 )
 
 // SetEmptyRoomCache 设置空教室缓存 key 为 date + campus + startTime + endTime，value直接采用bytes存储
@@ -14,18 +14,18 @@ func SetEmptyRoomCache(ctx context.Context, key string, emptyRoomList []string)
 	// 两天过期
 	err = RedisClient.Set(ctx, key, emptyRoomJson, constants.ClassroomKeyExpire).Err()
 	if err != nil {
-		utils.LoggerObj.Fatalf("dal.cache.SetEmptyRoomCache failed, err is %v", err)
+		logger.LoggerObj.Fatalf("dal.cache.SetEmptyRoomCache failed, err is %v", err)
 	}
 }
 func GetEmptyRoomCache(ctx context.Context, key string) (emptyRoomList []string) {
 	data, err := RedisClient.Get(ctx, key).Result()
 	if err != nil {
-		utils.LoggerObj.Fatalf("dal.cache.GetEmptyRoomCache failed, err is %v", err)
+		logger.LoggerObj.Fatalf("dal.cache.GetEmptyRoomCache failed, err is %v", err)
 		return nil
 	}
 	err = json.Unmarshal([]byte(data), &emptyRoomList)
 	if err != nil {
-		utils.LoggerObj.Fatalf("dal.cache.GetEmptyRoomCache Unmarshal failed, err is %v", err)
+		logger.LoggerObj.Fatalf("dal.cache.GetEmptyRoomCache Unmarshal failed, err is %v", err)
 		return nil
 	}
 	return

diff --git a/cmd/classroom/handler.go b/cmd/classroom/handler.go
@@ -5,7 +5,7 @@ import (
 	"github.com/west2-online/fzuhelper-server/cmd/classroom/pack"
 	"github.com/west2-online/fzuhelper-server/cmd/classroom/service"
 	classroom "github.com/west2-online/fzuhelper-server/kitex_gen/classroom"
-	"github.com/west2-online/fzuhelper-server/pkg/utils"
+	"github.com/west2-online/fzuhelper-server/pkg/logger"
 )
 
 // ClassroomServiceImpl implements the last service interface defined in the IDL.
@@ -22,6 +22,6 @@ func (s *ClassroomServiceImpl) GetEmptyRoom(ctx context.Context, req *classroom.
 	}
 	resp.Base = pack.BuildBaseResp(nil)
 	resp.Rooms = pack.BuildClassRooms(res, req.Campus)
-	utils.LoggerObj.Info("GetEmptyRoom success")
+	logger.LoggerObj.Info("GetEmptyRoom success")
 	return
 }
diff --git a/cmd/classroom/main.go b/cmd/classroom/main.go
@@ -12,6 +12,7 @@ import (
 	"github.com/west2-online/fzuhelper-server/config"
 	classroom "github.com/west2-online/fzuhelper-server/kitex_gen/classroom/classroomservice"
 	"github.com/west2-online/fzuhelper-server/pkg/constants"
+	"github.com/west2-online/fzuhelper-server/pkg/logger"
 	"github.com/west2-online/fzuhelper-server/pkg/utils"
 	"net"
 )
@@ -25,7 +26,7 @@ var (
 
 func Init() {
 	// config init
-	utils.LoggerInit()
+	logger.LoggerInit()
 	path = flag.String("config", "./config", "config path")
 	flag.Parse()
 	config.Init(*path, serviceName)

diff --git a/cmd/classroom/service/get_room.go b/cmd/classroom/service/get_room.go
@@ -5,7 +5,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/west2-online/fzuhelper-server/cmd/classroom/dal/cache"
 	"github.com/west2-online/fzuhelper-server/kitex_gen/classroom"
-	"github.com/west2-online/fzuhelper-server/pkg/utils"
+	"github.com/west2-online/fzuhelper-server/pkg/logger"
 	"github.com/west2-online/jwch"
 	"strings"
 )
@@ -51,7 +51,7 @@ func (s *ClassroomService) getQiShanEmptyRooms(req *classroom.EmptyRoomRequest)
 		End:    req.EndTime,
 	})
 	if err != nil {
-		utils.LoggerObj.Errorf("service.getQiShanEmptyRooms: %v", err)
+		logger.LoggerObj.Errorf("service.getQiShanEmptyRooms: %v", err)
 		return nil, errors.Wrap(err, "service.getQiShanEmptyRooms failed")
 	}
 	return rooms, nil
@@ -65,7 +65,7 @@ func (s *ClassroomService) getOtherEmptyRooms(req *classroom.EmptyRoomRequest) (
 		End:    req.EndTime,
 	})
 	if err != nil {
-		utils.LoggerObj.Errorf("service.getOtherEmptyRooms: %v", err)
+		logger.LoggerObj.Errorf("service.getOtherEmptyRooms: %v", err)
 		return nil, errors.Wrap(err, "service.getOtherEmptyRooms failed")
 	}
 

diff --git a/cmd/classroom/service/get_room_test.go b/cmd/classroom/service/get_room_test.go
@@ -7,7 +7,7 @@ import (
 	"github.com/west2-online/fzuhelper-server/cmd/classroom/dal"
 	"github.com/west2-online/fzuhelper-server/cmd/classroom/pack"
 	"github.com/west2-online/fzuhelper-server/kitex_gen/classroom"
-	"github.com/west2-online/fzuhelper-server/pkg/utils"
+	"github.com/west2-online/fzuhelper-server/pkg/logger"
 	"github.com/west2-online/jwch"
 	"net/http"
 	"testing"
@@ -20,7 +20,7 @@ func getIDAndCookies() (string, []*http.Cookie) {
 func Init() {
 	// config init
 
-	utils.LoggerInit()
+	logger.LoggerInit()
 	dal.Init()
 	klog.SetLevel(klog.LevelDebug)
 }

diff --git a/cmd/classroom/service/service.go b/cmd/classroom/service/service.go
@@ -5,8 +5,10 @@ import (
 	"github.com/west2-online/fzuhelper-server/config"
 	"github.com/west2-online/fzuhelper-server/kitex_gen/classroom"
 	"github.com/west2-online/fzuhelper-server/pkg/constants"
-	"github.com/west2-online/fzuhelper-server/pkg/utils"
+	"github.com/west2-online/fzuhelper-server/pkg/logger"
 	"github.com/west2-online/jwch"
+	"golang.org/x/time/rate"
+	"k8s.io/client-go/util/workqueue"
 	"net/http"
 	"strconv"
 	"time"
@@ -43,6 +45,21 @@ func CacheEmptyRooms() {
 		ctx := context.Background()
 		id, cookies := jwch.NewStudent().WithUser(config.DefaultUser.Account, config.DefaultUser.Password).GetIdentifierAndCookies()
 		l := NewClassroomService(ctx, id, cookies)
+		//使用具有限速功能的工作队列，避免教务处的压力过大
+		queue := workqueue.NewNamedRateLimitingQueue(
+			workqueue.NewMaxOfRateLimiter(
+				// For syncRec failures(i.e. doRecommend return err), the retry time is (2*minutes)*2^<num-failures>
+				// The maximum retry time is 24 hours
+				workqueue.NewItemExponentialFailureRateLimiter(constants.FailureRateLimiterBaseDelay, constants.FailureRateLimiterMaxDelay),
+				// 10 qps, 100 bucket size. This is only for retry speed, it's only the overall factor (not per item)
+				//每秒最多产生 10 个令牌（允许处理 10 个任务）。
+				//100：令牌桶最多存储 100 个令牌，允许积累的最大任务数量
+				&workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)},
+			),
+			constants.ClassroomService)
+		for i := 0; i < constants.NumWorkers; i++ {
+			go worker(queue, l)
+		}
 		var dates []string
 		currentTime := time.Now()
 		//设定一周时间
@@ -60,15 +77,40 @@ func CacheEmptyRooms() {
 							StartTime: strconv.Itoa(startTime),
 							EndTime:   strconv.Itoa(endTime),
 						}
-						go l.GetEmptyRooms(args)
-						//给3s的时间让服务器处理
-						time.Sleep(3 * time.Second)
+						queue.Add(args)
+						logger.LoggerObj.Debugf("classroom.service.CacheEmptyRooms add task %v", args)
 					}
 				}
-				utils.LoggerObj.Infof("Complete Data of campus %s", campus)
+				logger.LoggerObj.Infof("classroom.service.CacheEmptyRooms add all tasks of campus %v in the day %v", campus, date)
 			}
-			utils.LoggerObj.Infof("Complete Data of date %s", date)
 		}
-		time.Sleep(constants.ClassroomKeyExpire)
+		time.Sleep(constants.ScheduledTime)
+	}
+}
+
+// 从工作队列取出task并处理
+func worker(queue workqueue.RateLimitingInterface, l *ClassroomService) {
+	for {
+		task, shutDown := queue.Get()
+		if shutDown {
+			logger.LoggerObj.Debug("classroom.service.worker worker shutDown")
+			return
+		}
+		func(task any) {
+			defer queue.Done(task)
+			args, ok := task.(*classroom.EmptyRoomRequest)
+			if !ok {
+				logger.LoggerObj.Errorf("classroom.service.worker task type error: %T", task)
+				return
+			}
+			_, err := l.GetEmptyRooms(args)
+			if err != nil {
+				logger.LoggerObj.Errorf("classroom.service.worker GetEmptyRooms failed, args %v: %v", err, args)
+				return
+			}
+			//将任务标记为完成
+			queue.Forget(task)
+			logger.LoggerObj.Debug("classroom.service.worker task %v done", args)
+		}(task)
 	}
 }
diff --git a/cmd/user/handler.go b/cmd/user/handler.go
@@ -5,7 +5,7 @@ import (
 	"github.com/west2-online/fzuhelper-server/cmd/user/pack"
 	"github.com/west2-online/fzuhelper-server/cmd/user/service"
 	user "github.com/west2-online/fzuhelper-server/kitex_gen/user"
-	"github.com/west2-online/fzuhelper-server/pkg/utils"
+	"github.com/west2-online/fzuhelper-server/pkg/logger"
 )
 
 // UserServiceImpl implements the last service interface defined in the IDL.
@@ -23,6 +23,6 @@ func (s *UserServiceImpl) GetLoginData(ctx context.Context, req *user.GetLoginDa
 	resp.Base = pack.BuildBaseResp(nil)
 	resp.Id = id
 	resp.Cookies = cookies
-	utils.LoggerObj.Info("GetLoginData success")
+	logger.LoggerObj.Info("GetLoginData success")
 	return
 }
diff --git a/cmd/user/main.go b/cmd/user/main.go
@@ -10,6 +10,7 @@ import (
 	"github.com/west2-online/fzuhelper-server/config"
 	user "github.com/west2-online/fzuhelper-server/kitex_gen/user/userservice"
 	"github.com/west2-online/fzuhelper-server/pkg/constants"
+	"github.com/west2-online/fzuhelper-server/pkg/logger"
 	"github.com/west2-online/fzuhelper-server/pkg/utils"
 	"net"
 )
@@ -28,7 +29,7 @@ func Init() {
 	config.Init(*path, serviceName)
 
 	klog.SetLevel(klog.LevelDebug)
-	utils.LoggerInit()
+	logger.LoggerInit()
 }
 
 func main() {