refactor(controller): 优化Redis去重和缓存删除逻辑

2025-09-14 22:51:04 +08:00
parent 4c19f4b16c
commit 78d496ae72
3 changed files with 80 additions and 12 deletions
--- a/controller/dataController.go
+++ b/controller/dataController.go
@@ -54,7 +54,7 @@ func WriteDataHandler(c *gin.Context) {
 	}
 	dedupValue := strings.Split(input.Data, "----")[dataIndex[dedupObject]]

-	err = createCF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 100000000)
+	err = createCF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 100_000_000)
 	if err != nil && err.Error() != "ERR item exists" {
 		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
 		return
@@ -62,6 +62,7 @@ func WriteDataHandler(c *gin.Context) {
 	luaScript := `
 local dedupKey = KEYS[1]   -- KEYS[1]: 去重键 (dedup:token:object)
 local listKey = KEYS[2]    -- KEYS[2]: 列表键 (list:token)
+local deleteListKey = KEYS[3] -- KEYS[3]: 删除列表键 (delete-list:token)
 local dedupValue = ARGV[1] -- ARGV[1]: 去重值
 local rawData = ARGV[2]    -- ARGV[2]: 原始数据

@@ -78,6 +79,8 @@ redis.call('CF.ADD', dedupKey, dedupValue)
 -- 添加到列表
 redis.call('LPUSH', listKey, rawData)

+redis.call('LPUSH', deleteListKey, dedupValue)
+
 -- 返回成功结果
 return "ok"
 `
@@ -87,6 +90,7 @@ return "ok"
 		[]string{
 			fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject),
 			fmt.Sprintf("list:%s", input.Token),
+			fmt.Sprintf("delete-list:%s", input.Token),
 		},
 		dedupValue,
 		input.Data,
@@ -105,7 +109,7 @@ return "ok"
 }

 func createCF(bloomFilter string, capacity int64) error {
-	_, err := global.RDB.CFReserve(global.RCtx, bloomFilter, capacity).Result()
+	_, err := global.RDB.CFReserveBucketSize(global.RCtx, bloomFilter, capacity, 6).Result()
 	return err
 }

--- a/controller/tokenController.go
+++ b/controller/tokenController.go
@@ -5,9 +5,9 @@ import (
 	"dypid/global"
 	"net/http"
 	"strconv"
-	"strings"

 	"github.com/gin-gonic/gin"
+	"github.com/redis/go-redis/v9"
 )

 func ListTokenHandler(c *gin.Context) {
@@ -146,17 +146,29 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 		keys := global.RDB.Keys(global.RCtx, "dedup:"+input.Token+":*").Val()
 		global.RDB.Del(global.RCtx, keys...)
 	default:
-		//TODO 不考虑单独删除指定数量去重参考值
-		_, err := strconv.Atoi(input.DedupBF)
+		i, err := strconv.Atoi(input.DedupBF)
 		if err != nil {
 			c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_bf设置错误 " + err.Error()})
 			return
 		}
+		result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, 1, int64(i)).Val()
+		_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
+			for _, s := range result {
+				pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
+			}
+			return nil
+		})
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+		global.RDB.LTrim(global.RCtx, "delete-list:"+input.Token, int64(i), -1)
 	}
 	switch input.CacheList {
 	case "":
 	case "all":
 		global.RDB.Del(global.RCtx, "list:"+input.Token)
+		global.RDB.Del(global.RCtx, "delete-list:"+input.Token)
 	default:
 		i, err := strconv.Atoi(input.CacheList)
 		if err != nil {
@@ -174,13 +186,19 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 			c.JSON(http.StatusBadRequest, gin.H{"error": "both_number设置错误 " + err.Error()})
 			return
 		}
-		result := global.RDB.LRange(global.RCtx, "list:"+input.Token, 1, int64(i)).Val()
-		global.RDB.LTrim(global.RCtx, "list:"+input.Token, 1, int64(-i))
-		dataIndex, err := getDataIndex(input.Token)
-		for _, s := range result {
-			s2 := strings.Split(s, "----")[dataIndex[dedupObject]]
-			global.RDB.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s2)
+		result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, 1, int64(i)).Val()
+		_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
+			for _, s := range result {
+				pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
+			}
+			return nil
+		})
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
 		}
+		global.RDB.LTrim(global.RCtx, "delete-list:"+input.Token, int64(i), -1)
+		global.RDB.LTrim(global.RCtx, "list:"+input.Token, int64(i), -1)
 	}

 	//输出信息
--- a/web/src/views/TokenDetailView.vue
+++ b/web/src/views/TokenDetailView.vue
@@ -19,6 +19,10 @@ const options = ref([] as string[])
 const deleteSpecifyDataVisible = ref(false)
 const inputSpecifyData = ref('')

+const deleteSpecifyDedupVisible = ref(false)
+const inputSpecifyDedup = ref('')
+const deleteSpecifyRawVisible = ref(false)
+const inputSpecifyRaw = ref('')

 const getInfo = () => {
  if (value.value != '') {
@@ -76,13 +80,35 @@ const deleteSpecifyData = () => {
  axios.delete('/api/token/info', {
    params: {
      token: value.value,
-      cache_list: inputSpecifyData.value,
+      both_number: inputSpecifyData.value,
    }
  }).then(res => {
    getInfo()
    deleteSpecifyDataVisible.value = false
  })
 }
+const deleteSpecifyDedup = () => {
+  axios.delete('/api/token/info', {
+    params: {
+      token: value.value,
+      dedup_bf: inputSpecifyDedup.value,
+    }
+  }).then(res => {
+    getInfo()
+    deleteSpecifyDedupVisible.value = false
+  })
+}
+const deleteSpecifyRaw = () => {
+  axios.delete('/api/token/info', {
+    params: {
+      token: value.value,
+      cache_list: inputSpecifyRaw.value,
+    }
+  }).then(res => {
+    getInfo()
+    deleteSpecifyRawVisible.value = false
+  })
+}
 </script>


@@ -128,6 +154,10 @@ const deleteSpecifyData = () => {
    <p><b>管理</b></p>
    <el-button type="danger" @click="deleteDedup">删除全部去重参考值</el-button>
    <el-button type="danger" @click="deleteRedis">删除全部原始数据</el-button>
+    <div style="margin-top: 10px">
+      <el-button type="danger" @click="deleteSpecifyDedupVisible=true">删除指定数量去重参考值</el-button>
+      <el-button type="danger" @click="deleteSpecifyRawVisible=true">删除指定数量原始数据</el-button>
+    </div>
    <div style="margin-top: 10px">
      <el-button type="danger" @click="deleteSpecifyDataVisible=true">
        删除指定数量的数据（去重参考值+原始数据）
@@ -136,6 +166,22 @@ const deleteSpecifyData = () => {


    <!--弹窗输入-->
+    <el-dialog v-model="deleteSpecifyDedupVisible" title="删除指定数量去重参考值" width="400">
+      <el-input v-model="inputSpecifyDedup" style="width: 200px" placeholder="请输入删除数量"/>
+      <template #footer>
+        <el-button type="primary" @click="deleteSpecifyDedup">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
+    <el-dialog v-model="deleteSpecifyRawVisible" title="删除指定数量原始数据" width="400">
+      <el-input v-model="inputSpecifyRaw" style="width: 200px" placeholder="请输入删除数量"/>
+      <template #footer>
+        <el-button type="primary" @click="deleteSpecifyRaw">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
    <el-dialog v-model="deleteSpecifyDataVisible" title="删除指定数量的数据" width="400">
      <el-input v-model="inputSpecifyData" style="width: 200px" placeholder="请输入删除数量"/>
      <template #footer>