fix(数据): 修复删除数据过多而造成的redis连接超时问题

internal/controller/tokenController.go

@@ -171,23 +171,38 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 		global.RDB.Del(global.RCtx, keys...)
 		global.RDB.Del(global.RCtx, "delete-list:"+input.Token)
 	default:
-		i, err := strconv.Atoi(input.DedupBF)
+		num, err := strconv.Atoi(input.DedupBF)
 		if err != nil {
-			c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_bf设置错误 " + err.Error()})
+			c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_bf数量设置错误 " + err.Error()})
 			return
 		}
-		result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, int64(-i), -1).Val()
+
-		_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
+		for i := num; i > 0; i -= 20_0000 {
-			for _, s := range result {
+			a := 0
-				pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
+			if i > 20_0000 {
+				a = 20_0000
+			} else {
+				a = i
 			}
-			return nil
+
-		})
+			//获得要删的值（列表）
-		if err != nil {
+			result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, int64(-a), -1).Val()
-			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+
-			return
+			//用事件批量删除布隆过滤器中值
+			_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
+				for _, s := range result {
+					pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
+				}
+				return nil
+			})
+			if err != nil {
+				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+				return
+			}
+
+			//清除列表中要删的值
+			global.RDB.LTrim(global.RCtx, "delete-list:"+input.Token, 0, int64(-a-1))
 		}
-		global.RDB.LTrim(global.RCtx, "delete-list:"+input.Token, 0, int64(-i-1))
 	}
 
 	//删除原始数据
@@ -196,37 +211,46 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 	case "all":
 		global.RDB.Del(global.RCtx, "list:"+input.Token)
 	default:
-		i, err := strconv.Atoi(input.CacheList)
+		num, err := strconv.Atoi(input.CacheList)
 		if err != nil {
-			c.JSON(http.StatusBadRequest, gin.H{"error": "cache_list设置错误 " + err.Error()})
+			c.JSON(http.StatusBadRequest, gin.H{"error": "cache_list数量设置错误 " + err.Error()})
 			return
 		}
 
-		global.RDB.LTrim(global.RCtx, "list:"+input.Token, 0, int64(-i-1))
+		global.RDB.LTrim(global.RCtx, "list:"+input.Token, 0, int64(-num-1))
 	}
 
 	//删除去重参考值和原始数据
 	switch input.BothNumber {
 	case "":
 	default:
-		i, err := strconv.Atoi(input.BothNumber)
+		num, err := strconv.Atoi(input.BothNumber)
 		if err != nil {
-			c.JSON(http.StatusBadRequest, gin.H{"error": "both_number设置错误 " + err.Error()})
+			c.JSON(http.StatusBadRequest, gin.H{"error": "both_number数量设置错误 " + err.Error()})
 			return
 		}
 
-		result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, int64(-i), -1).Val()
+		for i := num; i > 0; i -= 20_0000 {
-		_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
+			a := 0
-			for _, s := range result {
+			if i > 20_0000 {
-				pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
+				a = 20_0000
+			} else {
+				a = i
+			}
+
+			result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, int64(-a), -1).Val()
+			_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
+				for _, s := range result {
+					pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
+				}
+				pipe.LTrim(global.RCtx, "delete-list:"+input.Token, 0, int64(-a-1))
+				pipe.LTrim(global.RCtx, "list:"+input.Token, 0, int64(-a-1))
+				return nil
+			})
+			if err != nil {
+				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+				return
 			}
-			pipe.LTrim(global.RCtx, "delete-list:"+input.Token, 0, int64(-i-1))
-			pipe.LTrim(global.RCtx, "list:"+input.Token, 0, int64(-i-1))
-			return nil
-		})
-		if err != nil {
-			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
-			return
 		}
 	}