更新 web/src/views/TokenManageView.vue

.gitea/workflows/build_tool.yaml

@@ -2,7 +2,7 @@ name: 构建上传工具
 on: [ push ]
 
 jobs:
-  build:
+  build-tool:
     env:
       RUNNER_TOOL_CACHE: /toolcache
     runs-on: ubuntu-latest
@@ -13,9 +13,7 @@ jobs:
       - name: 安装Go（镜像）
         run: |
           # 使用国内镜像下载 Go
-          export GO_MIRROR_URL="https://mirrors.aliyun.com/golang"
-          wget $GO_MIRROR_URL/go1.25.1.linux-amd64.tar.gz -O go.tar.gz
-          
+          wget https://mirrors.aliyun.com/golang/go1.25.1.linux-amd64.tar.gz -O go.tar.gz
           # 解压并设置环境变量
           sudo rm -rf /usr/local/go
           sudo tar -C /usr/local -xzf go.tar.gz
@@ -23,11 +21,6 @@ jobs:
         env:
           GOROOT: /usr/local/go
 
-#      - name: 安装Go
-#        uses: actions/setup-go@v5
-#        with:
-#          go-version-file: "go.mod"
-
       - name: 构建上传工具
         run: |
           go env -w CGO_ENABLED=0 \

.gitea/workflows/deploy-dev.yaml

@@ -2,7 +2,7 @@ name: 部署开发环境
 on: [ push ]
 
 jobs:
-  build-and-deploy:
+  deploy-dev:
     env:
       RUNNER_TOOL_CACHE: /toolcache
     runs-on: ubuntu-latest

.gitea/workflows/deploy-production.yaml

@@ -3,7 +3,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  build-and-deploy:
+  deploy-production:
     env:
       RUNNER_TOOL_CACHE: /toolcache
     runs-on: ubuntu-latest

controller/dataController.go

@@ -12,6 +12,7 @@ import (
 )
 
 func ReadDataHandler(c *gin.Context) {
+	//解析输入数据
 	input := struct {
 		Token string `form:"token" binding:"required"`
 	}{}
@@ -20,17 +21,17 @@ func ReadDataHandler(c *gin.Context) {
 		return
 	}
 
-	lLen := global.RDB.LLen(global.RCtx, fmt.Sprintf("list:%s", input.Token))
-	if lLen.Val() == 0 {
+	retData := global.RDB.LPop(global.RCtx, fmt.Sprintf("list:%s", input.Token)).Val()
+	if retData == "null" {
 		c.JSON(http.StatusOK, gin.H{"result": "数据库没有数据"})
 		return
 	}
-	retData := global.RDB.BLPop(global.RCtx, 0, fmt.Sprintf("list:%s", input.Token)).Val()[1]
 
 	c.String(http.StatusOK, retData)
 }
 
 func WriteDataHandler(c *gin.Context) {
+	//解析输入数据
 	input := struct {
 		Token string `form:"token" binding:"required"`
 		Data  string `form:"data" binding:"required"`
@@ -40,6 +41,7 @@ func WriteDataHandler(c *gin.Context) {
 		return
 	}
 
+	//数据获取
 	dedupObject, err := db.GetDedupObject(input.Token)
 	if err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
@@ -52,7 +54,7 @@ func WriteDataHandler(c *gin.Context) {
 	}
 	dedupValue := strings.Split(input.Data, "----")[dataIndex[dedupObject]]
 
-	err = createBF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 0.01, 100000000)
+	err = createCF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 100_000_000)
 	if err != nil && err.Error() != "ERR item exists" {
 		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
 		return
@@ -60,11 +62,12 @@ func WriteDataHandler(c *gin.Context) {
 	luaScript := `
 local dedupKey = KEYS[1]   -- KEYS[1]: 去重键 (dedup:token:object)
 local listKey = KEYS[2]    -- KEYS[2]: 列表键 (list:token)
+local deleteListKey = KEYS[3] -- KEYS[3]: 删除列表键 (delete-list:token)
 local dedupValue = ARGV[1] -- ARGV[1]: 去重值
 local rawData = ARGV[2]    -- ARGV[2]: 原始数据
 
 -- 检查布隆过滤器中是否已存在该值
-local exists = redis.call('BF.EXISTS', dedupKey, dedupValue)
+local exists = redis.call('CF.EXISTS', dedupKey, dedupValue)
 
 -- 如果已存在，返回已去重标记
 if exists == 1 then
@@ -72,10 +75,12 @@ if exists == 1 then
 end
 
 -- 添加到布隆过滤器
-redis.call('BF.ADD', dedupKey, dedupValue)
+redis.call('CF.ADD', dedupKey, dedupValue)
 -- 添加到列表
 redis.call('LPUSH', listKey, rawData)
 
+redis.call('LPUSH', deleteListKey, dedupValue)
+
 -- 返回成功结果
 return "ok"
 `
@@ -85,6 +90,7 @@ return "ok"
 		[]string{
 			fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject),
 			fmt.Sprintf("list:%s", input.Token),
+			fmt.Sprintf("delete-list:%s", input.Token),
 		},
 		dedupValue,
 		input.Data,
@@ -102,8 +108,8 @@ return "ok"
 	c.JSON(http.StatusInternalServerError, gin.H{"error": "WriteDataHandler 错误"})
 }
 
-func createBF(bloomFilter string, errorRate float64, capacity int64) error {
-	_, err := global.RDB.BFReserve(global.RCtx, bloomFilter, errorRate, capacity).Result()
+func createCF(bloomFilter string, capacity int64) error {
+	_, err := global.RDB.CFReserveBucketSize(global.RCtx, bloomFilter, capacity, 6).Result()
 	return err
 }
 

controller/tokenController.go

@@ -4,8 +4,10 @@ import (
 	"dypid/db"
 	"dypid/global"
 	"net/http"
+	"strconv"
 
 	"github.com/gin-gonic/gin"
+	"github.com/redis/go-redis/v9"
 )
 
 func ListTokenHandler(c *gin.Context) {
@@ -18,7 +20,7 @@ func CreateTokenHandler(c *gin.Context) {
 		Token       string `form:"token" binding:"required"`
 		DedupObject string `form:"dedup_object" binding:"required"`
 		DataFormat  string `form:"data_format" binding:"required"`
-		Notes       string `form:"notes" binding:"required"`
+		Notes       string `form:"notes"`
 	}{}
 	if err := c.ShouldBindQuery(&input); err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
@@ -41,7 +43,7 @@ func UpdateTokenHandler(c *gin.Context) {
 		Token       string `form:"token" binding:"required"`
 		DedupObject string `form:"dedup_object" binding:"required"`
 		DataFormat  string `form:"data_format" binding:"required"`
-		Notes       string `form:"notes" binding:"required"`
+		Notes       string `form:"notes"`
 	}{}
 	if err := c.ShouldBindQuery(&input); err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
@@ -105,7 +107,7 @@ func GetTokenInfoHandler(c *gin.Context) {
 	output.Token = input.Token
 	output.DedupObject = dedupObject
 	output.DataFormat = dataFormat
-	output.DedupItemsNumber = global.RDB.BFCard(global.RCtx, "dedup:"+input.Token+":"+dedupObject).Val()
+	output.DedupItemsNumber = global.RDB.CFInfo(global.RCtx, "dedup:"+input.Token+":"+dedupObject).Val().NumItemsInserted
 	output.CacheListNumber = global.RDB.LLen(global.RCtx, "list:"+input.Token).Val()
 
 	c.JSON(http.StatusOK, gin.H{"result": output})
@@ -115,8 +117,9 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 	//解析输入数据
 	input := struct {
 		Token      string `form:"token" binding:"required"`
-		DedupBF   bool   `form:"dedup_bf"`
-		CacheList bool   `form:"cache_list"`
+		DedupBF    string `form:"dedup_bf"`
+		CacheList  string `form:"cache_list"`
+		BothNumber string `form:"both_number"`
 	}{}
 	if err := c.ShouldBindQuery(&input); err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "Token不能为空"})
@@ -129,14 +132,77 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "Token不存在" + err.Error()})
 		return
 	}
+	dedupObject, err := db.GetDedupObject(input.Token)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
 
 	//删除去重对象
-	if input.DedupBF {
+	switch input.DedupBF {
+	case "":
+	case "all":
 		keys := global.RDB.Keys(global.RCtx, "dedup:"+input.Token+":*").Val()
 		global.RDB.Del(global.RCtx, keys...)
+		global.RDB.Del(global.RCtx, "delete-list:"+input.Token)
+	default:
+		i, err := strconv.Atoi(input.DedupBF)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_bf设置错误 " + err.Error()})
+			return
 		}
-	if input.CacheList {
+		result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, int64(-i), -1).Val()
+		_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
+			for _, s := range result {
+				pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
+			}
+			return nil
+		})
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+		global.RDB.LTrim(global.RCtx, "delete-list:"+input.Token, 0, int64(-i-1))
+	}
+
+	//删除原始数据
+	switch input.CacheList {
+	case "":
+	case "all":
 		global.RDB.Del(global.RCtx, "list:"+input.Token)
+	default:
+		i, err := strconv.Atoi(input.CacheList)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "cache_list设置错误 " + err.Error()})
+			return
+		}
+
+		global.RDB.LTrim(global.RCtx, "list:"+input.Token, 0, int64(-i-1))
+	}
+
+	//删除去重参考值和原始数据
+	switch input.BothNumber {
+	case "":
+	default:
+		i, err := strconv.Atoi(input.BothNumber)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "both_number设置错误 " + err.Error()})
+			return
+		}
+
+		result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, int64(-i), -1).Val()
+		_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
+			for _, s := range result {
+				pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
+			}
+			pipe.LTrim(global.RCtx, "delete-list:"+input.Token, 0, int64(-i-1))
+			pipe.LTrim(global.RCtx, "list:"+input.Token, 0, int64(-i-1))
+			return nil
+		})
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
 	}
 
 	//输出信息

tool/update-dypid.go

@@ -26,6 +26,41 @@ var httpClient = &http.Client{
 }
 
 func main() {
+	initConfig()
+
+	//检测./upload
+	fmt.Println("程序启动成功，正在检测txt文件")
+	for {
+		files, err := getTxtFiles("./")
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		if files != nil {
+			start := time.Now()
+
+			wg := sync.WaitGroup{}
+			for _, filePath := range files {
+				fmt.Println("正在上传文件:", filePath)
+				wg.Add(1)
+				go func() {
+					processFile(filePath)
+					err := os.Truncate(filePath, 0)
+					if err != nil {
+						fmt.Println("清空文件失败：", err)
+					}
+					wg.Done()
+				}()
+			}
+			wg.Wait()
+
+			fmt.Printf("上传完成，耗时：%s\n", time.Since(start))
+		}
+		time.Sleep(time.Minute)
+	}
+}
+
+func initConfig() {
 	//程序配置
 	viper.SetDefault("url", "http://localhost:8080")
 	viper.SetDefault("token", "")
@@ -45,33 +80,8 @@ func main() {
 			fmt.Errorf("无法读取配置文件: %w", err)
 		}
 	})
+}
 
-	//检测./upload
-	fmt.Println("程序启动成功，正在检测txt文件")
-	//os.Mkdir("./upload", os.ModePerm)
-	for {
-		files, err := getTxtFiles("./")
-		if err != nil {
-			fmt.Println(err)
-			return
-		}
-		if files != nil {
-			wg := sync.WaitGroup{}
-			for _, filePath := range files {
-				fmt.Println("正在上传文件:", filePath)
-				wg.Add(1)
-				go func() {
-					processFile(filePath)
-					os.Remove(filePath)
-					wg.Done()
-				}()
-			}
-			wg.Wait()
-
-		}
-		time.Sleep(2 * time.Second)
-	}
-}
 func uploadDataToServer(data string) error {
 	params := url.Values{}
 	params.Set("token", viper.GetString("token"))
@@ -88,10 +98,8 @@ func uploadDataToServer(data string) error {
 }
 
 // 获取目录中的所有txt文件
-func getTxtFiles(dir string) ([]string, error) {
-	var txtFiles []string
-
-	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+func getTxtFiles(dir string) (txtFiles []string, err error) {
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -151,6 +159,7 @@ func processFile(filePath string) {
 
 	if err := scanner.Err(); err != nil {
 		fmt.Printf("读取文件 %s 错误: %v\n", filePath, err)
+		return
 	}
 
 	fmt.Printf("文件【%s】处理完成，共处理 %d 行数据\n", filePath, lineCount)

web/src/views/TokenDetailView.vue

@@ -2,13 +2,26 @@
 import {useCounterStore} from "@/stores/counter.ts";
 import {ref, watch} from 'vue'
 import axios from "@/axios.ts";
-import {useRoute} from "vue-router"
 
 
-const route = useRoute()
-
+// 创建响应式引用，用于存储API请求结果
 const result = ref()
+
+// 创建响应式引用，用于存储当前选中的token值
 const value = ref('')
+value.value = useCounterStore().token
+
+// 创建响应式引用，用于存储下拉选项列表
+const options = ref([] as string[])
+
+// 控制删除指定Redis键的确认对话框的显示状态
+const deleteSpecifyDataVisible = ref(false)
+const inputSpecifyData = ref('')
+
+const deleteSpecifyDedupVisible = ref(false)
+const inputSpecifyDedup = ref('')
+const deleteSpecifyRawVisible = ref(false)
+const inputSpecifyRaw = ref('')
 
 const getInfo = () => {
   if (value.value != '') {
@@ -28,17 +41,18 @@ const deleteDedup = () => {
   axios.delete('/api/token/info', {
     params: {
       token: value.value,
-      dedup_bf: true
+      dedup_bf: "all"
     }
   }).then(res => {
     getInfo()
   })
 }
+
 const deleteRedis = () => {
   axios.delete('/api/token/info', {
     params: {
       token: value.value,
-      cache_list: true
+      cache_list: "all",
     }
   }).then(res => {
     getInfo()
@@ -49,23 +63,52 @@ getInfo()
 setInterval(getInfo, 5000)
 
 watch(value, (newValue) => {
+  useCounterStore().token = value.value
   getInfo()
 })
 
-interface optionsType {
-  value: string
-}
-
-const options = ref([] as optionsType[])
-value.value = useCounterStore().token
 
 axios.get('/api/token').then(res => {
   if (res.status == 200) {
     res.data.result.forEach((item: any) => {
-      options.value.push({"value": item.token})
+      options.value.push(item.token)
     })
   }
 })
+
+const deleteSpecifyData = () => {
+  axios.delete('/api/token/info', {
+    params: {
+      token: value.value,
+      both_number: inputSpecifyData.value,
+    }
+  }).then(res => {
+    getInfo()
+    deleteSpecifyDataVisible.value = false
+  })
+}
+const deleteSpecifyDedup = () => {
+  axios.delete('/api/token/info', {
+    params: {
+      token: value.value,
+      dedup_bf: inputSpecifyDedup.value,
+    }
+  }).then(res => {
+    getInfo()
+    deleteSpecifyDedupVisible.value = false
+  })
+}
+const deleteSpecifyRaw = () => {
+  axios.delete('/api/token/info', {
+    params: {
+      token: value.value,
+      cache_list: inputSpecifyRaw.value,
+    }
+  }).then(res => {
+    getInfo()
+    deleteSpecifyRawVisible.value = false
+  })
+}
 </script>
 
 
@@ -74,17 +117,19 @@ axios.get('/api/token').then(res => {
     <el-alert title="您没有权限访问此页面" type="error" center show-icon/>
   </div>
 
+
   <div v-if="useCounterStore().isAdmin">
     <b>当前Token：</b>
     <el-select v-model="value" placeholder="选择Token" style="width: 240px">
       <el-option
           v-for="item in options"
-          :key="item.value"
-          :value="item.value"
+          :key="item"
+          :value="item"
       />
     </el-select>
 
     <el-divider/>
+
     <b>Token信息（每5秒刷新）</b>
     <el-button type="primary" plain @click="getInfo">手动刷新</el-button>
     <el-descriptions
@@ -92,15 +137,59 @@ axios.get('/api/token').then(res => {
         :column="4"
         border
     >
-      <el-descriptions-item label="去重对象">{{ result?.dedup_object }}</el-descriptions-item>
-      <el-descriptions-item label="上传数据格式">{{ result?.data_format }}</el-descriptions-item>
-      <el-descriptions-item label="去重记录值">{{ result?.dedup_items_number }}</el-descriptions-item>
-      <el-descriptions-item label="Redis中数据条数">{{ result?.cache_list_number }}</el-descriptions-item>
+      <el-descriptions-item label="去重对象" label-width="150px">
+        <el-tag>{{ result?.dedup_object }}</el-tag>
+      </el-descriptions-item>
+      <el-descriptions-item label="上传数据格式" label-width="150px">
+        {{ result?.data_format }}
+      </el-descriptions-item>
+      <el-descriptions-item label="去重参考值数量" label-width="150px">
+        {{ result?.dedup_items_number }} 条
+      </el-descriptions-item>
+      <el-descriptions-item label="原始数据数量" label-width="150px">
+        {{ result?.cache_list_number }} 条
+      </el-descriptions-item>
     </el-descriptions>
 
     <p><b>管理</b></p>
-    <el-button type="danger" @click="deleteDedup">删除去重记录值</el-button>
-    <el-button type="danger" @click="deleteRedis">删除Redis数据</el-button>
+    <el-button type="danger" @click="deleteDedup">删除全部去重参考值</el-button>
+    <el-button type="danger" @click="deleteRedis">删除全部原始数据</el-button>
+    <div style="margin-top: 10px">
+      <el-button type="danger" @click="deleteSpecifyDedupVisible=true">删除指定数量去重参考值</el-button>
+      <el-button type="danger" @click="deleteSpecifyRawVisible=true">删除指定数量原始数据</el-button>
+    </div>
+    <div style="margin-top: 10px">
+      <el-button type="danger" @click="deleteSpecifyDataVisible=true">
+        删除指定数量的数据（去重参考值+原始数据）
+      </el-button>
+    </div>
+
+
+    <!--弹窗输入-->
+    <el-dialog v-model="deleteSpecifyDedupVisible" title="删除指定数量去重参考值" width="400">
+      <el-input v-model="inputSpecifyDedup" style="width: 200px" placeholder="请输入删除数量"/>
+      <template #footer>
+        <el-button type="primary" @click="deleteSpecifyDedup">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
+    <el-dialog v-model="deleteSpecifyRawVisible" title="删除指定数量原始数据" width="400">
+      <el-input v-model="inputSpecifyRaw" style="width: 200px" placeholder="请输入删除数量"/>
+      <template #footer>
+        <el-button type="primary" @click="deleteSpecifyRaw">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
+    <el-dialog v-model="deleteSpecifyDataVisible" title="删除指定数量的数据" width="400">
+      <el-input v-model="inputSpecifyData" style="width: 200px" placeholder="请输入删除数量"/>
+      <template #footer>
+        <el-button type="primary" @click="deleteSpecifyData">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
   </div>
 </template>
 

web/src/views/TokenManageView.vue

@@ -6,13 +6,16 @@ import {useCounterStore} from "@/stores/counter.ts";
 import {useRouter} from "vue-router";
 
 const tableData = ref([])
-
-axios.get("/api/token").then(res => {
-  tableData.value = res.data.result
-})
-
 const input = ref('')
 const value = ref('')
+const dataFormat = ref('')
+const inputPassWord = ref('')
+const router = useRouter()
+var rowOut: any
+const dedupObjectVisible = ref(false)
+const dataFormatVisible = ref(false)
+const inputNotes = ref("")
+const NotesVisible = ref(false)
 const options = [
   {
     value: 'uid',
@@ -31,15 +34,21 @@ const options = [
   }
 ]
 
-const dataFormat = ref('')
 const dataFormatOptions = [
   {
     value: 'uid----secid----pid----comment_id',
   }, {
     value: 'uid----secid',
+  }, {
+    value: 'dyid',
   }
 ]
 
+
+axios.get("/api/token").then(res => {
+  tableData.value = res.data.result
+})
+
 const addToken = () => {
   axios.post('/api/token', {}, {
     params: {
@@ -63,8 +72,6 @@ const addToken = () => {
   })
 }
 
-
-const router = useRouter()
 const viewDetails = (row: any) => {
   useCounterStore().token = row.token
   router.push({
@@ -72,13 +79,11 @@ const viewDetails = (row: any) => {
   })
 }
 
-var rowOut: any
-
-const dedupObjectVisible = ref(false)
 const dialogDedupObjectVisible = (row: any) => {
   rowOut = row
   dedupObjectVisible.value = true
 }
+
 const updateDedupObject = () => {
   dedupObjectVisible.value = false
   axios.put('/api/token', {}, {
@@ -103,11 +108,11 @@ const updateDedupObject = () => {
   })
 }
 
-const dataFormatVisible = ref(false)
 const dialogDataFormatVisible = (row: any) => {
   rowOut = row
   dataFormatVisible.value = true
 }
+
 const updateDataFormat = () => {
   dataFormatVisible.value = false
   axios.put('/api/token', {}, {
@@ -132,12 +137,11 @@ const updateDataFormat = () => {
   })
 }
 
-const inputNotes = ref("")
-const NotesVisible = ref(false)
 const dialogNotesVisible = (row: any) => {
   rowOut = row
   NotesVisible.value = true
 }
+
 const updateNotes = () => {
   NotesVisible.value = false
   axios.put('/api/token', {}, {
@@ -182,7 +186,6 @@ const deleteToken = (row: any) => {
   })
 }
 
-const inputPassWord = ref('')
 const checkPassword = () => {
   if (inputPassWord.value == "haha") {
     ElMessage({
@@ -228,7 +231,7 @@ const checkPassword = () => {
     <el-button type="primary" @click="addToken">添加Token</el-button>
 
     <!--Token列表-->
-    <el-table :data="tableData" style="width: 100%">
+    <el-table :data="tableData" stripe style="width: 100%">
       <el-table-column prop="token" label="Token" width="150"/>
       <el-table-column prop="dedup_object" label="去重对象" width="150"/>
       <el-table-column prop="data_format" label="上传数据格式" width="280"/>