refactor: 将布隆过滤器改为Cuckoo过滤器并优化代码结构

controller/dataController.go

@@ -31,6 +31,7 @@ func ReadDataHandler(c *gin.Context) {
 }
 
 func WriteDataHandler(c *gin.Context) {
+	//解析输入数据
 	input := struct {
 		Token string `form:"token" binding:"required"`
 		Data  string `form:"data" binding:"required"`
@@ -40,6 +41,7 @@ func WriteDataHandler(c *gin.Context) {
 		return
 	}
 
+	//数据获取
 	dedupObject, err := db.GetDedupObject(input.Token)
 	if err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
@@ -52,7 +54,7 @@ func WriteDataHandler(c *gin.Context) {
 	}
 	dedupValue := strings.Split(input.Data, "----")[dataIndex[dedupObject]]
 
-	err = createBF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 0.01, 100000000)
+	err = createCF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 100000000)
 	if err != nil && err.Error() != "ERR item exists" {
 		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
 		return
@@ -64,7 +66,7 @@ local dedupValue = ARGV[1] -- ARGV[1]: 去重值
 local rawData = ARGV[2]    -- ARGV[2]: 原始数据
 
 -- 检查布隆过滤器中是否已存在该值
-local exists = redis.call('BF.EXISTS', dedupKey, dedupValue)
+local exists = redis.call('CF.EXISTS', dedupKey, dedupValue)
 
 -- 如果已存在，返回已去重标记
 if exists == 1 then
@@ -72,7 +74,7 @@ if exists == 1 then
 end
 
 -- 添加到布隆过滤器
-redis.call('BF.ADD', dedupKey, dedupValue)
+redis.call('CF.ADD', dedupKey, dedupValue)
 -- 添加到列表
 redis.call('LPUSH', listKey, rawData)
 
@@ -102,8 +104,8 @@ return "ok"
 	c.JSON(http.StatusInternalServerError, gin.H{"error": "WriteDataHandler 错误"})
 }
 
-func createBF(bloomFilter string, errorRate float64, capacity int64) error {
-	_, err := global.RDB.BFReserve(global.RCtx, bloomFilter, errorRate, capacity).Result()
+func createCF(bloomFilter string, capacity int64) error {
+	_, err := global.RDB.CFReserve(global.RCtx, bloomFilter, capacity).Result()
 	return err
 }
 

controller/tokenController.go

@@ -3,9 +3,9 @@ package controller
 import (
 	"dypid/db"
 	"dypid/global"
-	"fmt"
 	"net/http"
 	"strconv"
+	"strings"
 
 	"github.com/gin-gonic/gin"
 )
@@ -119,6 +119,7 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 		Token      string `form:"token" binding:"required"`
 		DedupBF    string `form:"dedup_bf"`
 		CacheList  string `form:"cache_list"`
+		BothNumber string `form:"both_number"`
 	}{}
 	if err := c.ShouldBindQuery(&input); err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "Token不能为空"})
@@ -132,6 +133,12 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 		return
 	}
 
+	dedupObject, err := db.GetDedupObject(input.Token)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
 	//删除去重对象
 	switch input.DedupBF {
 	case "":
@@ -139,6 +146,7 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 		keys := global.RDB.Keys(global.RCtx, "dedup:"+input.Token+":*").Val()
 		global.RDB.Del(global.RCtx, keys...)
 	default:
+		//TODO 不考虑单独删除指定数量去重参考值
 		_, err := strconv.Atoi(input.DedupBF)
 		if err != nil {
 			c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_bf设置错误 " + err.Error()})
@@ -155,9 +163,25 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 			c.JSON(http.StatusBadRequest, gin.H{"error": "cache_list设置错误 " + err.Error()})
 			return
 		}
-		fmt.Println(-i)
 		global.RDB.LTrim(global.RCtx, "list:"+input.Token, 1, int64(-i))
 	}
+	//TODO
+	switch input.BothNumber {
+	case "":
+	default:
+		i, err := strconv.Atoi(input.BothNumber)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "both_number设置错误 " + err.Error()})
+			return
+		}
+		result := global.RDB.LRange(global.RCtx, "list:"+input.Token, 1, int64(i)).Val()
+		global.RDB.LTrim(global.RCtx, "list:"+input.Token, 1, int64(-i))
+		dataIndex, err := getDataIndex(input.Token)
+		for _, s := range result {
+			s2 := strings.Split(s, "----")[dataIndex[dedupObject]]
+			global.RDB.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s2)
+		}
+	}
 
 	//输出信息
 	c.JSON(http.StatusOK, gin.H{"result": "ok"})

tool/update-dypid.go

@@ -26,6 +26,41 @@ var httpClient = &http.Client{
 }
 
 func main() {
+	initConfig()
+
+	//检测./upload
+	fmt.Println("程序启动成功，正在检测txt文件")
+	for {
+		files, err := getTxtFiles("./")
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		if files != nil {
+			start := time.Now()
+
+			wg := sync.WaitGroup{}
+			for _, filePath := range files {
+				fmt.Println("正在上传文件:", filePath)
+				wg.Add(1)
+				go func() {
+					processFile(filePath)
+					err := os.Truncate(filePath, 0)
+					if err != nil {
+						fmt.Println("清空文件失败：", err)
+					}
+					wg.Done()
+				}()
+			}
+			wg.Wait()
+
+			fmt.Println("上传完成，耗时：", time.Since(start))
+		}
+		time.Sleep(time.Minute)
+	}
+}
+
+func initConfig() {
 	//程序配置
 	viper.SetDefault("url", "http://localhost:8080")
 	viper.SetDefault("token", "")
@@ -45,33 +80,6 @@ func main() {
 			fmt.Errorf("无法读取配置文件: %w", err)
 		}
 	})
-
-	//检测./upload
-	fmt.Println("程序启动成功，正在检测txt文件")
-	for {
-		files, err := getTxtFiles("./")
-		if err != nil {
-			fmt.Println(err)
-			return
-		}
-		if files != nil {
-			wg := sync.WaitGroup{}
-			for _, filePath := range files {
-				fmt.Println("正在上传文件:", filePath)
-				wg.Add(1)
-				go func() {
-					processFile(filePath)
-					err := os.Truncate(filePath, 0)
-					if err != nil {
-						fmt.Println("清空文件失败：", err)
-					}
-					wg.Done()
-				}()
-			}
-			wg.Wait()
-		}
-		time.Sleep(time.Minute)
-	}
 }
 
 func uploadDataToServer(data string) error {

web/src/views/TokenDetailView.vue

@@ -16,12 +16,8 @@ value.value = useCounterStore().token
 const options = ref([] as string[])
 
 // 控制删除指定Redis键的确认对话框的显示状态
-const deleteSpecifyRedisVisible = ref(false)
-const inputSpecifyRedis = ref('')
-
-// 控制删除指定去重项的确认对话框的显示状态
-const deleteSpecifyDedupVisible = ref(false)
-const inputSpecifyDedup = ref('')
+const deleteSpecifyDataVisible = ref(false)
+const inputSpecifyData = ref('')
 
 
 const getInfo = () => {
@@ -76,27 +72,15 @@ axios.get('/api/token').then(res => {
   }
 })
 
-const deleteSpecifyRedis = () => {
+const deleteSpecifyData = () => {
   axios.delete('/api/token/info', {
     params: {
       token: value.value,
-      cache_list: inputSpecifyRedis.value,
+      cache_list: inputSpecifyData.value,
     }
   }).then(res => {
     getInfo()
-    deleteSpecifyRedisVisible.value = false
-  })
-}
-
-const deleteSpecifyDedup = () => {
-  axios.delete('/api/token/info', {
-    params: {
-      token: value.value,
-      dedup_bf: inputSpecifyDedup.value,
-    }
-  }).then(res => {
-    getInfo()
-    deleteSpecifyDedupVisible.value = false
+    deleteSpecifyDataVisible.value = false
   })
 }
 </script>
@@ -107,6 +91,7 @@ const deleteSpecifyDedup = () => {
     <el-alert title="您没有权限访问此页面" type="error" center show-icon/>
   </div>
 
+
   <div v-if="useCounterStore().isAdmin">
     <b>当前Token：</b>
     <el-select v-model="value" placeholder="选择Token" style="width: 240px">
@@ -144,25 +129,17 @@ const deleteSpecifyDedup = () => {
     <el-button type="danger" @click="deleteDedup">删除全部去重参考值</el-button>
     <el-button type="danger" @click="deleteRedis">删除全部原始数据</el-button>
     <div style="margin-top: 10px">
-      <el-button type="danger" @click="deleteSpecifyDedupVisible=true">删除指定数量去重参考值</el-button>
-      <el-button type="danger" @click="deleteSpecifyRedisVisible=true">删除指定数量原始数据</el-button>
+      <el-button type="danger" @click="deleteSpecifyDataVisible=true">
+        删除指定数量的数据（去重参考值+原始数据）
+      </el-button>
     </div>
 
 
     <!--弹窗输入-->
-    <el-dialog v-model="deleteSpecifyDedupVisible" title="删除指定数量去重参考值" width="400">
-      <el-input v-model="inputSpecifyDedup" style="width: 200px" placeholder="请输入删除数量"/>
+    <el-dialog v-model="deleteSpecifyDataVisible" title="删除指定数量的数据" width="400">
+      <el-input v-model="inputSpecifyData" style="width: 200px" placeholder="请输入删除数量"/>
       <template #footer>
-        <el-button type="primary" @click="deleteSpecifyDedup">
-          确定
-        </el-button>
-      </template>
-    </el-dialog>
-
-    <el-dialog v-model="deleteSpecifyRedisVisible" title="删除指定数量原始数据" width="400">
-      <el-input v-model="inputSpecifyRedis" style="width: 200px" placeholder="请输入删除数量"/>
-      <template #footer>
-        <el-button type="primary" @click="deleteSpecifyRedis">
+        <el-button type="primary" @click="deleteSpecifyData">
           确定
         </el-button>
       </template>