Merge remote-tracking branch 'origin/main'

controller/dataController.go

@@ -12,6 +12,7 @@ import (
 )
 
 func ReadDataHandler(c *gin.Context) {
+	//解析输入数据
 	input := struct {
 		Token string `form:"token" binding:"required"`
 	}{}
@@ -20,12 +21,11 @@ func ReadDataHandler(c *gin.Context) {
 		return
 	}
 
-	lLen := global.RDB.LLen(global.RCtx, fmt.Sprintf("list:%s", input.Token))
-	if lLen.Val() == 0 {
+	retData := global.RDB.LPop(global.RCtx, fmt.Sprintf("list:%s", input.Token)).Val()
+	if retData == "null" {
 		c.JSON(http.StatusOK, gin.H{"result": "数据库没有数据"})
 		return
 	}
-	retData := global.RDB.BLPop(global.RCtx, 0, fmt.Sprintf("list:%s", input.Token)).Val()[1]
 
 	c.String(http.StatusOK, retData)
 }
@@ -54,7 +54,7 @@ func WriteDataHandler(c *gin.Context) {
 	}
 	dedupValue := strings.Split(input.Data, "----")[dataIndex[dedupObject]]
 
-	err = createCF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 100000000)
+	err = createCF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 100_000_000)
 	if err != nil && err.Error() != "ERR item exists" {
 		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
 		return
@@ -62,6 +62,7 @@ func WriteDataHandler(c *gin.Context) {
 	luaScript := `
 local dedupKey = KEYS[1]   -- KEYS[1]: 去重键 (dedup:token:object)
 local listKey = KEYS[2]    -- KEYS[2]: 列表键 (list:token)
+local deleteListKey = KEYS[3] -- KEYS[3]: 删除列表键 (delete-list:token)
 local dedupValue = ARGV[1] -- ARGV[1]: 去重值
 local rawData = ARGV[2]    -- ARGV[2]: 原始数据
 
@@ -78,6 +79,8 @@ redis.call('CF.ADD', dedupKey, dedupValue)
 -- 添加到列表
 redis.call('LPUSH', listKey, rawData)
 
+redis.call('LPUSH', deleteListKey, dedupValue)
+
 -- 返回成功结果
 return "ok"
 `
@@ -87,6 +90,7 @@ return "ok"
 		[]string{
 			fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject),
 			fmt.Sprintf("list:%s", input.Token),
+			fmt.Sprintf("delete-list:%s", input.Token),
 		},
 		dedupValue,
 		input.Data,
@@ -105,7 +109,7 @@ return "ok"
 }
 
 func createCF(bloomFilter string, capacity int64) error {
-	_, err := global.RDB.CFReserve(global.RCtx, bloomFilter, capacity).Result()
+	_, err := global.RDB.CFReserveBucketSize(global.RCtx, bloomFilter, capacity, 6).Result()
 	return err
 }
 

controller/tokenController.go

@@ -5,9 +5,9 @@ import (
 	"dypid/global"
 	"net/http"
 	"strconv"
-	"strings"
 
 	"github.com/gin-gonic/gin"
+	"github.com/redis/go-redis/v9"
 )
 
 func ListTokenHandler(c *gin.Context) {
@@ -26,6 +26,11 @@ func CreateTokenHandler(c *gin.Context) {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
 		return
 	}
+	//检查Token是否存在
+	if db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "创建Token失败，Token已经存在"})
+		return
+	}
 
 	//创建Token
 	err := db.CreateToken(input.Token, input.DedupObject, input.DataFormat, input.Notes)
@@ -49,6 +54,11 @@ func UpdateTokenHandler(c *gin.Context) {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
 		return
 	}
+	//检查Token是否存在
+	if !db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "更改失败，Token不存在"})
+		return
+	}
 
 	err := db.UpdateToken(input.Token, input.DedupObject, input.DataFormat, input.Notes)
 	if err != nil {
@@ -67,6 +77,11 @@ func DeleteTokenHandler(c *gin.Context) {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
 		return
 	}
+	//检查Token是否存在
+	if !db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "删除Token失败，Token不存在"})
+		return
+	}
 
 	err := db.DeleteToken(input.Token)
 	if err != nil {
@@ -85,6 +100,11 @@ func GetTokenInfoHandler(c *gin.Context) {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "Token不能为空"})
 		return
 	}
+	//检查Token是否存在
+	if !db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "获取信息失败，Token不存在"})
+		return
+	}
 
 	dedupObject, err := db.GetDedupObject(input.Token)
 	if err != nil {
@@ -107,7 +127,7 @@ func GetTokenInfoHandler(c *gin.Context) {
 	output.Token = input.Token
 	output.DedupObject = dedupObject
 	output.DataFormat = dataFormat
-	output.DedupItemsNumber = global.RDB.BFCard(global.RCtx, "dedup:"+input.Token+":"+dedupObject).Val()
+	output.DedupItemsNumber = global.RDB.CFInfo(global.RCtx, "dedup:"+input.Token+":"+dedupObject).Val().NumItemsInserted
 	output.CacheListNumber = global.RDB.LLen(global.RCtx, "list:"+input.Token).Val()
 
 	c.JSON(http.StatusOK, gin.H{"result": output})
@@ -125,6 +145,11 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "Token不能为空"})
 		return
 	}
+	//检查Token是否存在
+	if !db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "删除Token失败，Token不存在"})
+		return
+	}
 
 	//检查token是否存在
 	_, err := db.GetDedupObject(input.Token)
@@ -132,7 +157,6 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "Token不存在" + err.Error()})
 		return
 	}
-
 	dedupObject, err := db.GetDedupObject(input.Token)
 	if err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
@@ -145,14 +169,28 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 	case "all":
 		keys := global.RDB.Keys(global.RCtx, "dedup:"+input.Token+":*").Val()
 		global.RDB.Del(global.RCtx, keys...)
+		global.RDB.Del(global.RCtx, "delete-list:"+input.Token)
 	default:
-		//TODO 不考虑单独删除指定数量去重参考值
-		_, err := strconv.Atoi(input.DedupBF)
+		i, err := strconv.Atoi(input.DedupBF)
 		if err != nil {
 			c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_bf设置错误 " + err.Error()})
 			return
 		}
+		result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, int64(-i), -1).Val()
+		_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
+			for _, s := range result {
+				pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
 			}
+			return nil
+		})
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+		global.RDB.LTrim(global.RCtx, "delete-list:"+input.Token, 0, int64(-i-1))
+	}
+
+	//删除原始数据
 	switch input.CacheList {
 	case "":
 	case "all":
@@ -163,9 +201,11 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 			c.JSON(http.StatusBadRequest, gin.H{"error": "cache_list设置错误 " + err.Error()})
 			return
 		}
-		global.RDB.LTrim(global.RCtx, "list:"+input.Token, 1, int64(-i))
+
+		global.RDB.LTrim(global.RCtx, "list:"+input.Token, 0, int64(-i-1))
 	}
-	//TODO
+
+	//删除去重参考值和原始数据
 	switch input.BothNumber {
 	case "":
 	default:
@@ -174,12 +214,19 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 			c.JSON(http.StatusBadRequest, gin.H{"error": "both_number设置错误 " + err.Error()})
 			return
 		}
-		result := global.RDB.LRange(global.RCtx, "list:"+input.Token, 1, int64(i)).Val()
-		global.RDB.LTrim(global.RCtx, "list:"+input.Token, 1, int64(-i))
-		dataIndex, err := getDataIndex(input.Token)
+
+		result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, int64(-i), -1).Val()
+		_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
 			for _, s := range result {
-			s2 := strings.Split(s, "----")[dataIndex[dedupObject]]
-			global.RDB.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s2)
+				pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
+			}
+			pipe.LTrim(global.RCtx, "delete-list:"+input.Token, 0, int64(-i-1))
+			pipe.LTrim(global.RCtx, "list:"+input.Token, 0, int64(-i-1))
+			return nil
+		})
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
 		}
 	}
 

db/local.go

@@ -32,6 +32,16 @@ start:
 	}
 }
 
+func CheckToken(token string) bool {
+	InitLocalDB()
+	for _, t := range localDB {
+		if t.Token == token {
+			return true
+		}
+	}
+	return false
+}
+
 func ListToken() []Token {
 	InitLocalDB()
 	return localDB

tool/update-dypid.go

@@ -54,7 +54,7 @@ func main() {
 			}
 			wg.Wait()
 
-			fmt.Println("上传完成，耗时：", time.Since(start))
+			fmt.Printf("上传完成，耗时：%s\n", time.Since(start))
 		}
 		time.Sleep(time.Minute)
 	}

web/index.html

@@ -4,7 +4,7 @@
     <meta charset="UTF-8">
     <link rel="icon" href="/favicon.ico">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>dypid</title>
+    <title>抖音数据去重</title>
 </head>
 <body>
 <div id="app"></div>

web/src/views/TokenDetailView.vue

@@ -2,7 +2,6 @@
 import {useCounterStore} from "@/stores/counter.ts";
 import {ref, watch} from 'vue'
 import axios from "@/axios.ts";
-import {useRoute} from "vue-router"
 
 
 // 创建响应式引用，用于存储API请求结果
@@ -19,6 +18,10 @@ const options = ref([] as string[])
 const deleteSpecifyDataVisible = ref(false)
 const inputSpecifyData = ref('')
 
+const deleteSpecifyDedupVisible = ref(false)
+const inputSpecifyDedup = ref('')
+const deleteSpecifyRawVisible = ref(false)
+const inputSpecifyRaw = ref('')
 
 const getInfo = () => {
   if (value.value != '') {
@@ -60,6 +63,7 @@ getInfo()
 setInterval(getInfo, 5000)
 
 watch(value, (newValue) => {
+  useCounterStore().token = value.value
   getInfo()
 })
 
@@ -76,13 +80,35 @@ const deleteSpecifyData = () => {
   axios.delete('/api/token/info', {
     params: {
       token: value.value,
-      cache_list: inputSpecifyData.value,
+      both_number: inputSpecifyData.value,
     }
   }).then(res => {
     getInfo()
     deleteSpecifyDataVisible.value = false
   })
 }
+const deleteSpecifyDedup = () => {
+  axios.delete('/api/token/info', {
+    params: {
+      token: value.value,
+      dedup_bf: inputSpecifyDedup.value,
+    }
+  }).then(res => {
+    getInfo()
+    deleteSpecifyDedupVisible.value = false
+  })
+}
+const deleteSpecifyRaw = () => {
+  axios.delete('/api/token/info', {
+    params: {
+      token: value.value,
+      cache_list: inputSpecifyRaw.value,
+    }
+  }).then(res => {
+    getInfo()
+    deleteSpecifyRawVisible.value = false
+  })
+}
 </script>
 
 
@@ -128,6 +154,10 @@ const deleteSpecifyData = () => {
     <p><b>管理</b></p>
     <el-button type="danger" @click="deleteDedup">删除全部去重参考值</el-button>
     <el-button type="danger" @click="deleteRedis">删除全部原始数据</el-button>
+    <div style="margin-top: 10px">
+      <el-button type="danger" @click="deleteSpecifyDedupVisible=true">删除指定数量去重参考值</el-button>
+      <el-button type="danger" @click="deleteSpecifyRawVisible=true">删除指定数量原始数据</el-button>
+    </div>
     <div style="margin-top: 10px">
       <el-button type="danger" @click="deleteSpecifyDataVisible=true">
         删除指定数量的数据（去重参考值+原始数据）
@@ -136,6 +166,22 @@ const deleteSpecifyData = () => {
 
 
     <!--弹窗输入-->
+    <el-dialog v-model="deleteSpecifyDedupVisible" title="删除指定数量去重参考值" width="400">
+      <el-input v-model="inputSpecifyDedup" style="width: 200px" placeholder="请输入删除数量"/>
+      <template #footer>
+        <el-button type="primary" @click="deleteSpecifyDedup">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
+    <el-dialog v-model="deleteSpecifyRawVisible" title="删除指定数量原始数据" width="400">
+      <el-input v-model="inputSpecifyRaw" style="width: 200px" placeholder="请输入删除数量"/>
+      <template #footer>
+        <el-button type="primary" @click="deleteSpecifyRaw">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
     <el-dialog v-model="deleteSpecifyDataVisible" title="删除指定数量的数据" width="400">
       <el-input v-model="inputSpecifyData" style="width: 200px" placeholder="请输入删除数量"/>
       <template #footer>

web/src/views/TokenManageView.vue

@@ -39,6 +39,8 @@ const dataFormatOptions = [
     value: 'uid----secid----pid----comment_id',
   }, {
     value: 'uid----secid',
+  }, {
+    value: 'dyid',
   }
 ]