diff --git a/controller/dataController.go b/controller/dataController.go index 4ddd9b4..78a1f55 100644 --- a/controller/dataController.go +++ b/controller/dataController.go @@ -31,6 +31,7 @@ func ReadDataHandler(c *gin.Context) { } func WriteDataHandler(c *gin.Context) { + //解析输入数据 input := struct { Token string `form:"token" binding:"required"` Data string `form:"data" binding:"required"` @@ -40,6 +41,7 @@ func WriteDataHandler(c *gin.Context) { return } + //数据获取 dedupObject, err := db.GetDedupObject(input.Token) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) @@ -52,7 +54,7 @@ func WriteDataHandler(c *gin.Context) { } dedupValue := strings.Split(input.Data, "----")[dataIndex[dedupObject]] - err = createBF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 0.01, 100000000) + err = createCF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 100000000) if err != nil && err.Error() != "ERR item exists" { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return @@ -64,7 +66,7 @@ local dedupValue = ARGV[1] -- ARGV[1]: 去重值 local rawData = ARGV[2] -- ARGV[2]: 原始数据 -- 检查布隆过滤器中是否已存在该值 -local exists = redis.call('BF.EXISTS', dedupKey, dedupValue) +local exists = redis.call('CF.EXISTS', dedupKey, dedupValue) -- 如果已存在，返回已去重标记 if exists == 1 then @@ -72,7 +74,7 @@ if exists == 1 then end -- 添加到布隆过滤器 -redis.call('BF.ADD', dedupKey, dedupValue) +redis.call('CF.ADD', dedupKey, dedupValue) -- 添加到列表 redis.call('LPUSH', listKey, rawData) @@ -102,8 +104,8 @@ return "ok" c.JSON(http.StatusInternalServerError, gin.H{"error": "WriteDataHandler 错误"}) } -func createBF(bloomFilter string, errorRate float64, capacity int64) error { - _, err := global.RDB.BFReserve(global.RCtx, bloomFilter, errorRate, capacity).Result() +func createCF(bloomFilter string, capacity int64) error { + _, err := global.RDB.CFReserve(global.RCtx, bloomFilter, capacity).Result() return err } diff --git a/controller/tokenController.go b/controller/tokenController.go index d89007e..afa8f97 100644 --- a/controller/tokenController.go +++ b/controller/tokenController.go @@ -3,9 +3,9 @@ package controller import ( "dypid/db" "dypid/global" - "fmt" "net/http" "strconv" + "strings" "github.com/gin-gonic/gin" ) @@ -116,9 +116,10 @@ func GetTokenInfoHandler(c *gin.Context) { func DeleteTokenInfoHandler(c *gin.Context) { //解析输入数据 input := struct { - Token string `form:"token" binding:"required"` - DedupBF string `form:"dedup_bf"` - CacheList string `form:"cache_list"` + Token string `form:"token" binding:"required"` + DedupBF string `form:"dedup_bf"` + CacheList string `form:"cache_list"` + BothNumber string `form:"both_number"` }{} if err := c.ShouldBindQuery(&input); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": "Token不能为空"}) @@ -132,6 +133,12 @@ func DeleteTokenInfoHandler(c *gin.Context) { return } + dedupObject, err := db.GetDedupObject(input.Token) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + //删除去重对象 switch input.DedupBF { case "": @@ -139,6 +146,7 @@ func DeleteTokenInfoHandler(c *gin.Context) { keys := global.RDB.Keys(global.RCtx, "dedup:"+input.Token+":*").Val() global.RDB.Del(global.RCtx, keys...) default: + //TODO 不考虑单独删除指定数量去重参考值 _, err := strconv.Atoi(input.DedupBF) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_bf设置错误 " + err.Error()}) @@ -155,9 +163,25 @@ func DeleteTokenInfoHandler(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": "cache_list设置错误 " + err.Error()}) return } - fmt.Println(-i) global.RDB.LTrim(global.RCtx, "list:"+input.Token, 1, int64(-i)) } + //TODO + switch input.BothNumber { + case "": + default: + i, err := strconv.Atoi(input.BothNumber) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "both_number设置错误 " + err.Error()}) + return + } + result := global.RDB.LRange(global.RCtx, "list:"+input.Token, 1, int64(i)).Val() + global.RDB.LTrim(global.RCtx, "list:"+input.Token, 1, int64(-i)) + dataIndex, err := getDataIndex(input.Token) + for _, s := range result { + s2 := strings.Split(s, "----")[dataIndex[dedupObject]] + global.RDB.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s2) + } + } //输出信息 c.JSON(http.StatusOK, gin.H{"result": "ok"}) diff --git a/tool/update-dypid.go b/tool/update-dypid.go index 61a84bd..e6ac8b0 100644 --- a/tool/update-dypid.go +++ b/tool/update-dypid.go @@ -26,6 +26,41 @@ var httpClient = &http.Client{ } func main() { + initConfig() + + //检测./upload + fmt.Println("程序启动成功，正在检测txt文件") + for { + files, err := getTxtFiles("./") + if err != nil { + fmt.Println(err) + return + } + if files != nil { + start := time.Now() + + wg := sync.WaitGroup{} + for _, filePath := range files { + fmt.Println("正在上传文件:", filePath) + wg.Add(1) + go func() { + processFile(filePath) + err := os.Truncate(filePath, 0) + if err != nil { + fmt.Println("清空文件失败：", err) + } + wg.Done() + }() + } + wg.Wait() + + fmt.Println("上传完成，耗时：", time.Since(start)) + } + time.Sleep(time.Minute) + } +} + +func initConfig() { //程序配置 viper.SetDefault("url", "http://localhost:8080") viper.SetDefault("token", "") @@ -45,33 +80,6 @@ func main() { fmt.Errorf("无法读取配置文件: %w", err) } }) - - //检测./upload - fmt.Println("程序启动成功，正在检测txt文件") - for { - files, err := getTxtFiles("./") - if err != nil { - fmt.Println(err) - return - } - if files != nil { - wg := sync.WaitGroup{} - for _, filePath := range files { - fmt.Println("正在上传文件:", filePath) - wg.Add(1) - go func() { - processFile(filePath) - err := os.Truncate(filePath, 0) - if err != nil { - fmt.Println("清空文件失败：", err) - } - wg.Done() - }() - } - wg.Wait() - } - time.Sleep(time.Minute) - } } func uploadDataToServer(data string) error { diff --git a/web/src/views/TokenDetailView.vue b/web/src/views/TokenDetailView.vue index 7f55477..4f79b2e 100644 --- a/web/src/views/TokenDetailView.vue +++ b/web/src/views/TokenDetailView.vue @@ -16,12 +16,8 @@ value.value = useCounterStore().token const options = ref([] as string[]) // 控制删除指定Redis键的确认对话框的显示状态 -const deleteSpecifyRedisVisible = ref(false) -const inputSpecifyRedis = ref('') - -// 控制删除指定去重项的确认对话框的显示状态 -const deleteSpecifyDedupVisible = ref(false) -const inputSpecifyDedup = ref('') +const deleteSpecifyDataVisible = ref(false) +const inputSpecifyData = ref('') const getInfo = () => { @@ -76,27 +72,15 @@ axios.get('/api/token').then(res => { } }) -const deleteSpecifyRedis = () => { +const deleteSpecifyData = () => { axios.delete('/api/token/info', { params: { token: value.value, - cache_list: inputSpecifyRedis.value, + cache_list: inputSpecifyData.value, } }).then(res => { getInfo() - deleteSpecifyRedisVisible.value = false - }) -} - -const deleteSpecifyDedup = () => { - axios.delete('/api/token/info', { - params: { - token: value.value, - dedup_bf: inputSpecifyDedup.value, - } - }).then(res => { - getInfo() - deleteSpecifyDedupVisible.value = false + deleteSpecifyDataVisible.value = false }) } @@ -107,6 +91,7 @@ const deleteSpecifyDedup = () => { +
当前Token： @@ -144,25 +129,17 @@ const deleteSpecifyDedup = () => { 删除全部去重参考值 删除全部原始数据
- 删除指定数量去重参考值 - 删除指定数量原始数据 + + 删除指定数量的数据（去重参考值+原始数据） +
- - + + - - - - -