Merge remote-tracking branch 'origin/main'

.gitea/workflows/build_tool.yaml

@@ -0,0 +1,37 @@
+name: 构建上传工具
+on: [ push ]
+
+jobs:
+  build-tool:
+    env:
+      RUNNER_TOOL_CACHE: /toolcache
+    runs-on: ubuntu-latest
+    steps:
+      - name: 检出代码
+        uses: actions/checkout@v4
+
+      - name: 安装Go（镜像）
+        run: |
+          # 使用国内镜像下载 Go
+          wget https://mirrors.aliyun.com/golang/go1.25.1.linux-amd64.tar.gz -O go.tar.gz
+          # 解压并设置环境变量
+          sudo rm -rf /usr/local/go
+          sudo tar -C /usr/local -xzf go.tar.gz
+          echo "/usr/local/go/bin" >> $GITHUB_PATH
+        env:
+          GOROOT: /usr/local/go
+
+      - name: 构建上传工具
+        run: |
+          go env -w CGO_ENABLED=0 \
+          && go env -w GOARCH=amd64 \
+          && go env -w GOOS=windows \
+          && go mod tidy \
+          && cd ./tool \
+          && go build -o 上传工具.exe
+
+      - name: 上传构建文件
+        uses: actions/upload-artifact@v3
+        with:
+          name: 上传工具
+          path: tool/上传工具.exe

.gitea/workflows/deploy-dev.yaml

@@ -0,0 +1,39 @@
+name: 部署开发环境
+on: [ push ]
+
+jobs:
+  deploy-dev:
+    env:
+      RUNNER_TOOL_CACHE: /toolcache
+    runs-on: ubuntu-latest
+    steps:
+      - name: 检出代码
+        uses: actions/checkout@v4
+
+      - name: 构建Docker镜像
+        run: docker build -t dypid:latest .
+
+      - name: 导出镜像
+        run: mkdir release && docker save -o release/dypid.tar dypid:latest && docker rmi dypid:latest
+
+      - name: 安装sshpass工具
+        run: |
+          apt update
+          apt install -y sshpass
+
+      - name: 部署程序到测试服务器
+        env:
+          SSH_HOST: ${{ secrets.TEST_SSH_HOST }}
+          SSH_USER: ${{ secrets.TEST_SSH_USER }}
+          SSH_PASSWORD: ${{ secrets.TEST_SSH_PASSWORD }}
+          DEPLOY_DIR: "/data/dypid"
+        run: |
+          set -ex
+          # 上传新镜像
+          sshpass -p "$SSH_PASSWORD" scp -o StrictHostKeyChecking=no release/dypid.tar $SSH_USER@$SSH_HOST:/tmp/dypid.tar
+          # 重启程序
+          sshpass -p "$SSH_PASSWORD" ssh -o StrictHostKeyChecking=no $SSH_USER@$SSH_HOST \
+          "docker load -i /tmp/dypid.tar \
+          && cd $DEPLOY_DIR \
+          && docker compose up -d \
+          && docker image prune -f"

.gitea/workflows/deploy-production.yaml

@@ -0,0 +1,40 @@
+name: 部署生产环境
+on:
+  workflow_dispatch:
+
+jobs:
+  deploy-production:
+    env:
+      RUNNER_TOOL_CACHE: /toolcache
+    runs-on: ubuntu-latest
+    steps:
+      - name: 检出代码
+        uses: actions/checkout@v4
+
+      - name: 构建Docker镜像
+        run: docker build -t dypid:latest .
+
+      - name: 导出镜像
+        run: mkdir release && docker save -o release/dypid.tar dypid:latest && docker rmi dypid:latest
+
+      - name: 安装sshpass工具
+        run: |
+          apt update
+          apt install -y sshpass
+
+      - name: 部署程序到生产服务器
+        env:
+          SSH_HOST: ${{ secrets.SSH_HOST }}
+          SSH_USER: ${{ secrets.SSH_USER }}
+          SSH_PASSWORD: ${{ secrets.SSH_PASSWORD }}
+          DEPLOY_DIR: "/data/dypid"
+        run: |
+          set -ex
+          # 上传新镜像
+          sshpass -p "$SSH_PASSWORD" scp -o StrictHostKeyChecking=no release/dypid.tar $SSH_USER@$SSH_HOST:/tmp/dypid.tar
+          # 重启程序
+          sshpass -p "$SSH_PASSWORD" ssh -o StrictHostKeyChecking=no $SSH_USER@$SSH_HOST \
+          "docker load -i /tmp/dypid.tar \
+          && cd $DEPLOY_DIR \
+          && docker compose up -d \
+          && docker image prune -f"

Dockerfile

@@ -0,0 +1,34 @@
+FROM node:lts AS webBuilder
+
+WORKDIR /build
+
+COPY web .
+
+RUN npm install \
+    && npm run build
+
+FROM golang AS goBuilder
+
+WORKDIR /build
+
+COPY . .
+COPY --from=webBuilder /build/dist ./web/dist
+
+RUN go env -w CGO_ENABLED=0 \
+    && go mod tidy \
+    && go build -o dypid
+
+FROM alpine:latest
+
+WORKDIR /data
+
+COPY --from=goBuilder /build/dypid /dypid
+
+RUN apk update \
+    && apk add --no-cache tzdata \
+    && cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+    && echo "Asia/Shanghai" > /etc/timezone \
+    && apk del tzdata \
+    && chmod +x /dypid
+
+ENTRYPOINT [ "/dypid" ]

README.md

@@ -1,2 +1,3 @@
 # dypid
 
+一个高性能高并发数据去重工具

controller/dataController.go

@@ -3,55 +3,71 @@ package controller
 import (
 	"dypid/db"
 	"dypid/global"
-	"dypid/model"
-	"encoding/json"
 	"fmt"
+	"net/http"
+	"strings"
 
 	"github.com/gin-gonic/gin"
 	"github.com/redis/go-redis/v9"
 )
 
 func ReadDataHandler(c *gin.Context) {
-	lLen := global.RDB.LLen(global.RCtx, fmt.Sprintf("list:%s", c.Query("token")))
-	if lLen.Val() == 0 {
-		c.JSON(200, gin.H{"result": "数据库没有数据"})
+	//解析输入数据
+	input := struct {
+		Token string `form:"token" binding:"required"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
 		return
 	}
-	retData := global.RDB.BLPop(global.RCtx, 0, fmt.Sprintf("list:%s", c.Query("token")))
-	newData := model.Data{}
-	err := json.Unmarshal([]byte(retData.Val()[1]), &newData)
-	if err != nil {
-		c.JSON(400, gin.H{"error": err.Error()})
+
+	retData := global.RDB.LPop(global.RCtx, fmt.Sprintf("list:%s", input.Token)).Val()
+	if retData == "null" {
+		c.JSON(http.StatusOK, gin.H{"result": "数据库没有数据"})
 		return
 	}
-	c.JSON(200, gin.H{"result": newData})
+
+	c.String(http.StatusOK, retData)
 }
+
 func WriteDataHandler(c *gin.Context) {
-	data := model.Data{}
-
-	if err := c.BindQuery(&data); err != nil {
-		c.JSON(400, gin.H{"error": err.Error()})
+	//解析输入数据
+	input := struct {
+		Token string `form:"token" binding:"required"`
+		Data  string `form:"data" binding:"required"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
 		return
 	}
-	dedupObject, err := db.GetDedupObject(data.Token)
+
+	//数据获取
+	dedupObject, err := db.GetDedupObject(input.Token)
 	if err != nil {
-		c.JSON(400, gin.H{"error": err.Error()})
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
 		return
 	}
-	err = createBF(fmt.Sprintf("dedup:%s:%s", data.Token, dedupObject), 0.01, 100000000)
-	if err != nil && err.Error() != "ERR item exists" {
-		c.JSON(400, gin.H{"error": err.Error()})
+	dataIndex, err := getDataIndex(input.Token)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
 		return
 	}
+	dedupValue := strings.Split(input.Data, "----")[dataIndex[dedupObject]]
 
+	err = createCF(fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject), 100_000_000)
+	if err != nil && err.Error() != "ERR item exists" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
 	luaScript := `
 local dedupKey = KEYS[1]   -- KEYS[1]: 去重键 (dedup:token:object)
 local listKey = KEYS[2]    -- KEYS[2]: 列表键 (list:token)
+local deleteListKey = KEYS[3] -- KEYS[3]: 删除列表键 (delete-list:token)
 local dedupValue = ARGV[1] -- ARGV[1]: 去重值
-local jsonData = ARGV[2]   -- ARGV[2]: JSON序列化的数据
+local rawData = ARGV[2]    -- ARGV[2]: 原始数据
 
 -- 检查布隆过滤器中是否已存在该值
-local exists = redis.call('BF.EXISTS', dedupKey, dedupValue)
+local exists = redis.call('CF.EXISTS', dedupKey, dedupValue)
 
 -- 如果已存在，返回已去重标记
 if exists == 1 then
@@ -59,43 +75,54 @@ if exists == 1 then
 end
 
 -- 添加到布隆过滤器
-redis.call('BF.ADD', dedupKey, dedupValue)
+redis.call('CF.ADD', dedupKey, dedupValue)
 -- 添加到列表
-redis.call('LPUSH', listKey, jsonData)
+redis.call('LPUSH', listKey, rawData)
+
+redis.call('LPUSH', deleteListKey, dedupValue)
 
 -- 返回成功结果
 return "ok"
 `
-	k1 := fmt.Sprintf("dedup:%s:%s", data.Token, dedupObject)
-	k2 := fmt.Sprintf("list:%s", data.Token)
-	v1 := c.Query(dedupObject)
-	v2, err := json.Marshal(data)
-	if err != nil {
-		c.JSON(500, gin.H{"error": "JSON序列化失败 " + err.Error()})
-		return
-	}
-
 	result, err := redis.NewScript(luaScript).Run(
 		global.RCtx,
 		global.RDB,
-		[]string{k1, k2},
-		v1,
-		string(v2),
+		[]string{
+			fmt.Sprintf("dedup:%s:%s", input.Token, dedupObject),
+			fmt.Sprintf("list:%s", input.Token),
+			fmt.Sprintf("delete-list:%s", input.Token),
+		},
+		dedupValue,
+		input.Data,
 	).Result()
 	if err != nil {
-		c.JSON(500, gin.H{"error": "Redis操作失败 " + err.Error()})
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Redis操作失败 " + err.Error()})
 		return
 	}
 
 	if resultMap, ok := result.(string); ok {
-		c.JSON(200, gin.H{"result": resultMap})
+		c.JSON(http.StatusOK, gin.H{"result": resultMap})
 		return
 	}
 
-	c.JSON(500, gin.H{"error": "WriteDataHandler 错误"})
+	c.JSON(http.StatusInternalServerError, gin.H{"error": "WriteDataHandler 错误"})
 }
 
-func createBF(bloomFilter string, errorRate float64, capacity int64) error {
-	_, err := global.RDB.BFReserve(global.RCtx, bloomFilter, errorRate, capacity).Result()
+func createCF(bloomFilter string, capacity int64) error {
+	_, err := global.RDB.CFReserveBucketSize(global.RCtx, bloomFilter, capacity, 6).Result()
 	return err
 }
+
+func getDataIndex(token string) (index map[string]int, err error) {
+	dataFormat, err := db.GetDataFormat(token)
+	if err != nil {
+		return nil, err
+	}
+
+	f := strings.Split(dataFormat, "----")
+	index = make(map[string]int)
+	for i, s := range f {
+		index[s] = i
+	}
+	return index, nil
+}

controller/tokenController.go

@@ -4,8 +4,10 @@ import (
 	"dypid/db"
 	"dypid/global"
 	"net/http"
+	"strconv"
 
 	"github.com/gin-gonic/gin"
+	"github.com/redis/go-redis/v9"
 )
 
 func ListTokenHandler(c *gin.Context) {
@@ -13,50 +15,80 @@ func ListTokenHandler(c *gin.Context) {
 }
 
 func CreateTokenHandler(c *gin.Context) {
-	if c.Query("token") == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "token不能为空"})
+	//解析输入数据
+	input := struct {
+		Token       string `form:"token" binding:"required"`
+		DedupObject string `form:"dedup_object" binding:"required"`
+		DataFormat  string `form:"data_format" binding:"required"`
+		Notes       string `form:"notes"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
 		return
-	} else if c.Query("dedup_object") == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_object不能为空"})
+	}
+	//检查Token是否存在
+	if db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "创建Token失败，Token已经存在"})
 		return
 	}
 
-	err := db.CreateToken(c.Query("token"), c.Query("dedup_object"))
+	//创建Token
+	err := db.CreateToken(input.Token, input.DedupObject, input.DataFormat, input.Notes)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
+
+	//返回
 	c.JSON(http.StatusOK, gin.H{"result": "ok"})
 }
 
 func UpdateTokenHandler(c *gin.Context) {
-	if c.Query("token") == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "token不能为空"})
+	input := struct {
+		Token       string `form:"token" binding:"required"`
+		DedupObject string `form:"dedup_object" binding:"required"`
+		DataFormat  string `form:"data_format" binding:"required"`
+		Notes       string `form:"notes"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
 		return
-	} else if c.Query("dedup_object") == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_object不能为空"})
+	}
+	//检查Token是否存在
+	if !db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "更改失败，Token不存在"})
 		return
 	}
 
-	err := db.UpdateToken(c.Query("token"), c.Query("dedup_object"))
+	err := db.UpdateToken(input.Token, input.DedupObject, input.DataFormat, input.Notes)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
+
 	c.JSON(http.StatusOK, gin.H{"result": "ok"})
 }
 
 func DeleteTokenHandler(c *gin.Context) {
-	if c.Query("token") == "" {
-		c.JSON(http.StatusBadRequest, gin.H{"error": "token不能为空"})
+	input := struct {
+		Token string `form:"token" binding:"required"`
+	}{}
+	if err := c.ShouldBindQuery(&input); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
+		return
+	}
+	//检查Token是否存在
+	if !db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "删除Token失败，Token不存在"})
 		return
 	}
 
-	err := db.DeleteToken(c.Query("token"))
+	err := db.DeleteToken(input.Token)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
 		return
 	}
+
 	c.JSON(http.StatusOK, gin.H{"result": "ok"})
 }
 
@@ -68,22 +100,34 @@ func GetTokenInfoHandler(c *gin.Context) {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "Token不能为空"})
 		return
 	}
+	//检查Token是否存在
+	if !db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "获取信息失败，Token不存在"})
+		return
+	}
 
 	dedupObject, err := db.GetDedupObject(input.Token)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "获取去重对象失败 " + err.Error()})
 		return
 	}
+	dataFormat, err := db.GetDataFormat(input.Token)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "获取数据格式失败 " + err.Error()})
+		return
+	}
 
 	output := struct {
 		Token            string `json:"token"`
 		DedupObject      string `json:"dedup_object"`
+		DataFormat       string `json:"data_format"`
 		DedupItemsNumber int64  `json:"dedup_items_number"`
 		CacheListNumber  int64  `json:"cache_list_number"`
 	}{}
 	output.Token = input.Token
 	output.DedupObject = dedupObject
-	output.DedupItemsNumber = global.RDB.BFCard(global.RCtx, "dedup:"+input.Token+":"+dedupObject).Val()
+	output.DataFormat = dataFormat
+	output.DedupItemsNumber = global.RDB.CFInfo(global.RCtx, "dedup:"+input.Token+":"+dedupObject).Val().NumItemsInserted
 	output.CacheListNumber = global.RDB.LLen(global.RCtx, "list:"+input.Token).Val()
 
 	c.JSON(http.StatusOK, gin.H{"result": output})
@@ -93,13 +137,19 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 	//解析输入数据
 	input := struct {
 		Token      string `form:"token" binding:"required"`
-		DedupBF   bool   `form:"dedup_bf"`
-		CacheList bool   `form:"cache_list"`
+		DedupBF    string `form:"dedup_bf"`
+		CacheList  string `form:"cache_list"`
+		BothNumber string `form:"both_number"`
 	}{}
 	if err := c.ShouldBindQuery(&input); err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "Token不能为空"})
 		return
 	}
+	//检查Token是否存在
+	if !db.CheckToken(input.Token) {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "删除Token失败，Token不存在"})
+		return
+	}
 
 	//检查token是否存在
 	_, err := db.GetDedupObject(input.Token)
@@ -107,14 +157,77 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "Token不存在" + err.Error()})
 		return
 	}
+	dedupObject, err := db.GetDedupObject(input.Token)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
 
 	//删除去重对象
-	if input.DedupBF {
+	switch input.DedupBF {
+	case "":
+	case "all":
 		keys := global.RDB.Keys(global.RCtx, "dedup:"+input.Token+":*").Val()
 		global.RDB.Del(global.RCtx, keys...)
+		global.RDB.Del(global.RCtx, "delete-list:"+input.Token)
+	default:
+		i, err := strconv.Atoi(input.DedupBF)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_bf设置错误 " + err.Error()})
+			return
 		}
-	if input.CacheList {
+		result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, int64(-i), -1).Val()
+		_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
+			for _, s := range result {
+				pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
+			}
+			return nil
+		})
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+		global.RDB.LTrim(global.RCtx, "delete-list:"+input.Token, 0, int64(-i-1))
+	}
+
+	//删除原始数据
+	switch input.CacheList {
+	case "":
+	case "all":
 		global.RDB.Del(global.RCtx, "list:"+input.Token)
+	default:
+		i, err := strconv.Atoi(input.CacheList)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "cache_list设置错误 " + err.Error()})
+			return
+		}
+
+		global.RDB.LTrim(global.RCtx, "list:"+input.Token, 0, int64(-i-1))
+	}
+
+	//删除去重参考值和原始数据
+	switch input.BothNumber {
+	case "":
+	default:
+		i, err := strconv.Atoi(input.BothNumber)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "both_number设置错误 " + err.Error()})
+			return
+		}
+
+		result := global.RDB.LRange(global.RCtx, "delete-list:"+input.Token, int64(-i), -1).Val()
+		_, err = global.RDB.TxPipelined(global.RCtx, func(pipe redis.Pipeliner) error {
+			for _, s := range result {
+				pipe.CFDel(global.RCtx, "dedup:"+input.Token+":"+dedupObject, s)
+			}
+			pipe.LTrim(global.RCtx, "delete-list:"+input.Token, 0, int64(-i-1))
+			pipe.LTrim(global.RCtx, "list:"+input.Token, 0, int64(-i-1))
+			return nil
+		})
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
 	}
 
 	//输出信息

db/local.go

@@ -12,6 +12,8 @@ var localDB []Token
 type Token struct {
 	Token       string `json:"token"`
 	DedupObject string `json:"dedup_object"`
+	DataFormat  string `json:"data_format"`
+	Notes       string `json:"notes"`
 }
 
 func InitLocalDB() {
@@ -30,12 +32,22 @@ start:
 	}
 }
 
+func CheckToken(token string) bool {
+	InitLocalDB()
+	for _, t := range localDB {
+		if t.Token == token {
+			return true
+		}
+	}
+	return false
+}
+
 func ListToken() []Token {
 	InitLocalDB()
 	return localDB
 }
 
-func CreateToken(token string, dedupObject string) error {
+func CreateToken(token string, dedupObject string, dataFormat string, Notes string) error {
 	InitLocalDB()
 	for _, t := range localDB {
 		if t.Token == token {
@@ -43,7 +55,7 @@ func CreateToken(token string, dedupObject string) error {
 		}
 	}
 
-	a := append(localDB, Token{token, dedupObject})
+	a := append(localDB, Token{token, dedupObject, dataFormat, Notes})
 	marshal, err := json.Marshal(a)
 	if err != nil {
 		return err
@@ -55,12 +67,14 @@ func CreateToken(token string, dedupObject string) error {
 	return nil
 }
 
-func UpdateToken(token string, dedupObject string) error {
+func UpdateToken(token string, dedupObject string, dataFormat string, Notes string) error {
 	InitLocalDB()
 
 	for i, t := range localDB {
 		if t.Token == token {
 			localDB[i].DedupObject = dedupObject
+			localDB[i].DataFormat = dataFormat
+			localDB[i].Notes = Notes
 
 			marshal, err := json.Marshal(localDB)
 			if err != nil {
@@ -72,6 +86,7 @@ func UpdateToken(token string, dedupObject string) error {
 			return nil
 		}
 	}
+
 	InitLocalDB()
 	return errors.New("token not found")
 }
@@ -93,6 +108,7 @@ func DeleteToken(token string) error {
 			return nil
 		}
 	}
+
 	InitLocalDB()
 	return errors.New("token not found")
 }
@@ -105,3 +121,12 @@ func GetDedupObject(token string) (dedupObject string, err error) {
 	}
 	return "", errors.New("未找到Token")
 }
+
+func GetDataFormat(token string) (dataFormat string, err error) {
+	for i, t := range localDB {
+		if t.Token == token {
+			return localDB[i].DataFormat, nil
+		}
+	}
+	return "", errors.New("未找到Token")
+}

go.mod

@@ -3,7 +3,9 @@ module dypid
 go 1.25
 
 require (
-	github.com/labstack/echo/v4 v4.13.4
+	github.com/fsnotify/fsnotify v1.8.0
+	github.com/gin-contrib/cors v1.7.6
+	github.com/gin-gonic/gin v1.10.1
 	github.com/redis/go-redis/v9 v9.12.1
 	github.com/spf13/viper v1.20.1
 )
@@ -13,14 +15,9 @@ require (
 	github.com/bytedance/sonic/loader v0.2.4 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudwego/base64x v0.1.5 // indirect
-	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.9 // indirect
-	github.com/gin-contrib/cors v1.7.6 // indirect
 	github.com/gin-contrib/sse v1.1.0 // indirect
-	github.com/gin-contrib/static v1.1.5 // indirect
-	github.com/gin-gonic/gin v1.10.1 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.26.0 // indirect
@@ -28,9 +25,7 @@ require (
 	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
-	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
-	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -43,14 +38,12 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.3.0 // indirect
-	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/valyala/fasttemplate v1.2.2 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
 	golang.org/x/arch v0.18.0 // indirect
 	golang.org/x/crypto v0.39.0 // indirect
 	golang.org/x/net v0.41.0 // indirect
-	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/sys v0.35.0 // indirect
 	golang.org/x/text v0.26.0 // indirect
 	google.golang.org/protobuf v1.36.6 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -2,23 +2,15 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
-github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
-github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
-github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ=
-github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4=
 github.com/bytedance/sonic v1.13.3 h1:MS8gmaH16Gtirygw7jV91pDCN33NyMrPbN7qiYhEsF0=
 github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4=
-github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
 github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY=
 github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
-github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
 github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
 github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
-github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
 github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -29,47 +21,32 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
 github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
-github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
-github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
-github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
-github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
 github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY=
 github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok=
 github.com/gin-contrib/cors v1.7.6 h1:3gQ8GMzs1Ylpf70y8bMw4fVpycXIeX1ZemuSQIsnQQY=
 github.com/gin-contrib/cors v1.7.6/go.mod h1:Ulcl+xN4jel9t1Ry8vqph23a60FwH9xVLd+3ykmTjOk=
-github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
-github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
-github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E=
-github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0=
 github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
 github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
-github.com/gin-contrib/static v1.1.5 h1:bAPqT4KTZN+4uDY1b90eSrD1t8iNzod7Jj8njwmnzz4=
-github.com/gin-contrib/static v1.1.5/go.mod h1:8JSEXwZHcQ0uCrLPcsvnAJ4g+ODxeupP8Zetl9fd8wM=
 github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ=
 github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
-github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
-github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k=
 github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
 github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
 github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
-github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
-github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
 github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
-github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
 github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
@@ -77,14 +54,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/labstack/echo/v4 v4.13.4 h1:oTZZW+T3s9gAu5L8vmzihV7/lkXGZuITzTQkTEhcXEA=
-github.com/labstack/echo/v4 v4.13.4/go.mod h1:g63b33BZ5vZzcIUF8AtRH40DrTlXnx4UMC8rBdndmjQ=
-github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
-github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
-github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
-github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -92,8 +63,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
-github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
 github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -117,64 +86,40 @@ github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqj
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
-github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
-github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA=
 github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
-github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
-github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
 go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
-golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
-golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
-golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
-golang.org/x/arch v0.16.0 h1:foMtLTdyOmIniqWCHjY6+JxuC54XP1fDwx4N0ASyW+U=
-golang.org/x/arch v0.16.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
 golang.org/x/arch v0.18.0 h1:WN9poc33zL4AzGxqf8VtpKUnGvMi8O9lhNyBMF/85qc=
 golang.org/x/arch v0.18.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk=
-golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
-golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
 golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
 golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
-golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
-golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
 golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
 golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
-golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
 golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
-google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
-google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
 google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
-rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

main.go

@@ -6,9 +6,10 @@ import (
 	"dypid/db"
 	"embed"
 	"fmt"
+	"io/fs"
+	"net/http"
 
 	"github.com/gin-contrib/cors"
-	"github.com/gin-contrib/static"
 	"github.com/gin-gonic/gin"
 )
 
@@ -25,12 +26,17 @@ func main() {
 	//跨域设置
 	r.Use(cors.Default())
 
-	//静态文件目录
-	web, err := static.EmbedFolder(webDir, "web/dist")
-	if err != nil {
-		panic(err)
-	}
-	r.Use(static.Serve("/", web))
+	//Vue网站服务
+	assets, _ := fs.Sub(webDir, "web/dist/assets")
+	r.StaticFS("/assets", http.FS(assets))
+	icon, _ := fs.ReadFile(webDir, "web/dist/favicon.ico")
+	r.GET("/favicon.ico", func(c *gin.Context) {
+		c.Data(200, "image/x-icon", icon)
+	})
+	indexHtml, _ := fs.ReadFile(webDir, "web/dist/index.html")
+	r.NoRoute(func(c *gin.Context) {
+		c.Data(200, "text/html; charset=utf-8", indexHtml)
+	})
 
 	//API接口
 	g := r.Group("/api") //初始化路由组 /api/xxxx
@@ -48,12 +54,10 @@ func main() {
 	}
 
 	// 监听并在 0.0.0.0:8080 上启动服务
-	fmt.Printf("服务器正在%s运行\n", config.APPConfig.Host)
-	err = r.Run(config.APPConfig.Host)
+	fmt.Printf("服务器正在运行：http://%s\n", config.APPConfig.Host)
+	err := r.Run(config.APPConfig.Host)
 	if err != nil {
 		fmt.Println("服务启动失败：", err)
 		return
 	}
 }
-
-//atomic.AddInt64(&Same, 1)

model/data.go

@@ -4,7 +4,7 @@ type Data struct {
 	Token     string `form:"token" json:"token"`
 	Uid       string `form:"uid" json:"uid"`
 	Pid       string `form:"pid" json:"pid"`
-	SecId     string `form:"secid" json:"secId"`
+	SecId     string `form:"secid" json:"secid"`
 	Dyid      string `form:"dyid" json:"dyid"`
 	Key       string `form:"key" json:"key"`
 	CommentId string `form:"comment_id" json:"comment_id"`

tool/update-dypid.go

@@ -16,31 +16,52 @@ import (
 	"github.com/spf13/viper"
 )
 
-type UploadData struct {
-	Dyid      string // dyid
-	Uid       string // uid
-	Secuid    string // secuid
-	Pid       string // pid
-	CommentId string // comment_id
-	Id1       string // id1
-	Id2       string // id2
-	Id3       string // id3
-	Id4       string // id4
-	Id5       string // id5
-}
-
 var httpClient = &http.Client{
 	Transport: &http.Transport{
-		MaxIdleConns:        100,
-		MaxIdleConnsPerHost: 10,
+		MaxIdleConns:        200,
+		MaxIdleConnsPerHost: 100,
 		IdleConnTimeout:     30 * time.Second,
 	},
 	Timeout: 30 * time.Second,
 }
 
 func main() {
+	initConfig()
+
+	//检测./upload
+	fmt.Println("程序启动成功，正在检测txt文件")
+	for {
+		files, err := getTxtFiles("./")
+		if err != nil {
+			fmt.Println(err)
+			return
+		}
+		if files != nil {
+			start := time.Now()
+
+			wg := sync.WaitGroup{}
+			for _, filePath := range files {
+				fmt.Println("正在上传文件:", filePath)
+				wg.Add(1)
+				go func() {
+					processFile(filePath)
+					err := os.Truncate(filePath, 0)
+					if err != nil {
+						fmt.Println("清空文件失败：", err)
+					}
+					wg.Done()
+				}()
+			}
+			wg.Wait()
+
+			fmt.Printf("上传完成，耗时：%s\n", time.Since(start))
+		}
+		time.Sleep(time.Minute)
+	}
+}
+
+func initConfig() {
 	//程序配置
-	//设置默认值
 	viper.SetDefault("url", "http://localhost:8080")
 	viper.SetDefault("token", "")
 	viper.SetDefault("thread-count", 10)
@@ -59,57 +80,26 @@ func main() {
 			fmt.Errorf("无法读取配置文件: %w", err)
 		}
 	})
+}
 
-	//检测./upload
-	fmt.Println("程序启动成功，正在检测upload")
-	os.Mkdir("./upload", os.ModePerm)
-	for {
-		files, err := getTxtFiles("./upload")
-		if err != nil {
-			fmt.Println(err)
-			return
-		}
-		if files != nil {
-			wg := sync.WaitGroup{}
-			for _, filePath := range files {
-				fmt.Println("正在上传文件:", filePath)
-				wg.Add(1)
-				go func() {
-					processFile(filePath)
-					os.Remove(filePath)
-					wg.Done()
-				}()
-			}
-			wg.Wait()
-
-		}
-		time.Sleep(2 * time.Second)
-	}
-}
-func uploadDataToServer(data UploadData) error {
+func uploadDataToServer(data string) error {
 	params := url.Values{}
-	params.Add("token", viper.GetString("token"))
-	params.Add("dyid", data.Dyid)
-	params.Add("uid", data.Uid)
-	params.Add("secuid", data.Secuid)
-	params.Add("pid", data.Pid)
-	params.Add("comment_id", data.CommentId)
-	params.Add("id1", data.Id1)
-	params.Add("id2", data.Id2)
-	params.Add("id3", data.Id3)
-	params.Add("id4", data.Id4)
-	params.Add("id5", data.Id5)
+	params.Set("token", viper.GetString("token"))
+	params.Set("data", data)
 
 	resp, err := httpClient.Post(viper.GetString("url")+"/api/data?"+params.Encode(), "application/x-www-form-urlencoded", strings.NewReader(""))
+	if err != nil {
+		return err
+	}
+	if resp != nil {
 		_, _ = io.Copy(io.Discard, resp.Body)
+	}
 	return err
 }
 
 // 获取目录中的所有txt文件
-func getTxtFiles(dir string) ([]string, error) {
-	var txtFiles []string
-
-	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+func getTxtFiles(dir string) (txtFiles []string, err error) {
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -159,6 +149,9 @@ func processFile(filePath string) {
 		line := scanner.Text()
 		lines <- line
 		lineCount++
+		if lineCount%10000 == 0 {
+			fmt.Printf("文件【%s】处理进度：%v%%\n", filePath, float64(lineCount)/40000*100)
+		}
 	}
 
 	close(lines)
@@ -166,9 +159,10 @@ func processFile(filePath string) {
 
 	if err := scanner.Err(); err != nil {
 		fmt.Printf("读取文件 %s 错误: %v\n", filePath, err)
+		return
 	}
 
-	fmt.Printf("文件 %s 处理完成，共处理 %d 行数据\n", filePath, lineCount)
+	fmt.Printf("文件【%s】处理完成，共处理 %d 行数据\n", filePath, lineCount)
 }
 
 func processLines(lines <-chan string, workerID int, filePath string) {
@@ -177,39 +171,9 @@ func processLines(lines <-chan string, workerID int, filePath string) {
 		if strings.TrimSpace(line) == "" {
 			continue
 		}
-
-		// 使用----分割行数据
-		parts := strings.Split(line, "----")
-		for i := 0; i < 10; i++ {
-			parts = append(parts, "")
-		}
-
-		// 确保有足够的字段
-		if len(parts) < 3 {
-			fmt.Printf("Worker %d (文件 %s): 行数据字段不足，跳过: %s\n", workerID, filePath, line)
-			continue
-		}
-
-		// 创建上传数据结构
-		uploadData := UploadData{
-			Dyid:      parts[0],
-			Uid:       parts[1],
-			Secuid:    parts[2],
-			Pid:       parts[3],
-			CommentId: parts[4],
-			Id1:       parts[5],
-			Id2:       parts[6],
-			Id3:       parts[7],
-			Id4:       parts[8],
-			Id5:       parts[9],
-		}
-
 		// 上传数据
-		if err := uploadDataToServer(uploadData); err != nil {
+		if err := uploadDataToServer(line); err != nil {
 			fmt.Printf("Worker %d (文件 %s): 上传失败: %v\n", workerID, filePath, err)
 		}
-
-		// 添加短暂延迟避免服务器过载
-		//time.Sleep(10 * time.Millisecond)
 	}
 }

web/index.html

@@ -4,7 +4,7 @@
     <meta charset="UTF-8">
     <link rel="icon" href="/favicon.ico">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Vite App</title>
+    <title>抖音数据去重</title>
 </head>
 <body>
 <div id="app"></div>

web/package-lock.json

@@ -11,6 +11,7 @@
         "axios": "^1.11.0",
         "element-plus": "^2.11.1",
         "pinia": "^3.0.3",
+        "pinia-plugin-persistedstate": "^4.5.0",
         "vue": "^3.5.18",
         "vue-router": "^4.5.1"
       },
@@ -2083,6 +2084,12 @@
         }
       }
     },
+    "node_modules/deep-pick-omit": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/deep-pick-omit/-/deep-pick-omit-1.2.1.tgz",
+      "integrity": "sha512-2J6Kc/m3irCeqVG42T+SaUMesaK7oGWaedGnQQK/+O0gYc+2SP5bKh/KKTE7d7SJ+GCA9UUE1GRzh6oDe0EnGw==",
+      "license": "MIT"
+    },
     "node_modules/default-browser": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.2.1.tgz",
@@ -2126,6 +2133,12 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/defu": {
+      "version": "6.1.4",
+      "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.4.tgz",
+      "integrity": "sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==",
+      "license": "MIT"
+    },
     "node_modules/delayed-stream": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@@ -2135,6 +2148,12 @@
         "node": ">=0.4.0"
       }
     },
+    "node_modules/destr": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/destr/-/destr-2.0.5.tgz",
+      "integrity": "sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==",
+      "license": "MIT"
+    },
     "node_modules/dunder-proto": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -3074,6 +3093,33 @@
         }
       }
     },
+    "node_modules/pinia-plugin-persistedstate": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/pinia-plugin-persistedstate/-/pinia-plugin-persistedstate-4.5.0.tgz",
+      "integrity": "sha512-QTkP1xJVyCdr2I2p3AKUZM84/e+IS+HktRxKGAIuDzkyaKKV48mQcYkJFVVDuvTxlI5j6X3oZObpqoVB8JnWpw==",
+      "license": "MIT",
+      "dependencies": {
+        "deep-pick-omit": "^1.2.1",
+        "defu": "^6.1.4",
+        "destr": "^2.0.5"
+      },
+      "peerDependencies": {
+        "@nuxt/kit": ">=3.0.0",
+        "@pinia/nuxt": ">=0.10.0",
+        "pinia": ">=3.0.0"
+      },
+      "peerDependenciesMeta": {
+        "@nuxt/kit": {
+          "optional": true
+        },
+        "@pinia/nuxt": {
+          "optional": true
+        },
+        "pinia": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/postcss": {
       "version": "8.5.6",
       "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz",

web/package.json

@@ -17,6 +17,7 @@
     "axios": "^1.11.0",
     "element-plus": "^2.11.1",
     "pinia": "^3.0.3",
+    "pinia-plugin-persistedstate": "^4.5.0",
     "vue": "^3.5.18",
     "vue-router": "^4.5.1"
   },

web/src/App.vue

@@ -1,59 +1,15 @@
 <script setup lang="ts">
-import {ref, watch} from 'vue'
-import {useRoute, useRouter} from "vue-router";
 
-const router = useRouter()
-const route = useRoute()
-
-const activeIndex = ref(route.name?.toString() || "TokenList")
-
-watch(route, (newRoute) => {
-  activeIndex.value = newRoute.name?.toString() || "TokenList"
-})
-
-const handleSelect = (key: string) => {
-  router.push({
-    name: key
-  })
-}
 </script>
 
-
 <template>
-  <el-container>
-    <el-header>
-      <el-menu
-          :default-active="activeIndex"
-          class="el-menu-demo"
-          mode="horizontal"
-          @select="handleSelect"
-      >
-        <el-menu-item index="TokenList">Token列表</el-menu-item>
-        <el-menu-item index="TokenDetail">Token信息</el-menu-item>
-      </el-menu>
-    </el-header>
-
-    <el-container>
-      <el-main>
   <router-view></router-view>
-      </el-main>
-    </el-container>
-  </el-container>
 </template>
 
-
-<style>
+<style scoped>
 html, body, #app {
   height: 100%;
   margin: 0;
   padding: 0;
 }
-
-.el-container {
-  height: 100%;
-}
-
-.el-menu-demo {
-  line-height: 60px;
-}
 </style>

web/src/main.ts

@@ -1,5 +1,6 @@
 import {createApp} from 'vue'
 import {createPinia} from 'pinia'
+import persistedState from 'pinia-plugin-persistedstate';
 
 import App from './App.vue'
 import router from './router'
@@ -8,7 +9,7 @@ import 'element-plus/dist/index.css'
 
 const app = createApp(App)
 
-app.use(createPinia())
+app.use(createPinia().use(persistedState))
 app.use(router)
 app.use(ElementPlus)
 

web/src/router/index.ts

@@ -1,11 +1,34 @@
 import {createRouter, createWebHistory} from 'vue-router'
 
-import TokenListView from '@/views/TokenListView.vue'
+import TokenManageView from '@/views/TokenManageView.vue'
 import TokenDetailView from '@/views/TokenDetailView.vue'
+import AdminView from '@/views/AdminView.vue'
+import HomeView from '@/views/HomeView.vue'
 
 const routes = [
-    {path: '/', name: "TokenList", component: TokenListView},
-    {path: '/', name: "TokenDetail", component: TokenDetailView},
+    {
+        path: '/',
+        name: "Home",
+        component: HomeView,
+    },
+    {
+        path: '/haha',
+        name: "Admin",
+        component: AdminView,
+        children: [
+            {
+                path: '',
+                name: "TokenManage",
+                component: TokenManageView
+            },
+            {
+                path: 'token',
+                name: "TokenDetail",
+                component: TokenDetailView
+            },
+        ],
+    },
+
 ];
 
 const router = createRouter({

web/src/stores/counter.ts

@@ -2,7 +2,14 @@ import {ref, computed} from 'vue'
 import {defineStore} from 'pinia'
 
 export const useCounterStore = defineStore('counter', () => {
+    const homeToken = ref("")
+
     const token = ref("")
 
-    return {token}
+    const isAdmin = ref(false)
+
+
+    return {token, isAdmin}
+}, {
+    persist: true
 })

web/src/views/AdminView.vue

@@ -0,0 +1,50 @@
+<script setup lang="ts">
+import {ref, watch} from 'vue'
+import {useRoute, useRouter} from "vue-router";
+import {useCounterStore} from "@/stores/counter.ts";
+
+const router = useRouter()
+const route = useRoute()
+
+const activeIndex = ref(route.name?.toString() || "TokenList")
+
+watch(route, (newRoute) => {
+  activeIndex.value = newRoute.name?.toString() || "TokenList"
+})
+
+const handleSelect = (key: string) => {
+  router.push({
+    name: key
+  })
+}
+</script>
+
+<template>
+  <el-container>
+    <el-header>
+      <el-menu
+          :default-active="activeIndex"
+          class="el-menu-demo"
+          mode="horizontal"
+          @select="handleSelect"
+      >
+        <el-menu-item index="TokenManage">管理Token</el-menu-item>
+        <el-menu-item index="TokenDetail">Token详细信息</el-menu-item>
+        <el-menu-item v-if="useCounterStore().isAdmin">
+          <el-button type="danger" plain @click="useCounterStore().isAdmin=false">退出管理员</el-button>
+        </el-menu-item>
+      </el-menu>
+
+    </el-header>
+
+    <el-container>
+      <el-main>
+        <router-view></router-view>
+      </el-main>
+    </el-container>
+  </el-container>
+</template>
+
+<style scoped>
+
+</style>

web/src/views/HomeView.vue

@@ -0,0 +1,77 @@
+<script setup lang="ts">
+import {ref} from 'vue'
+import axios from "@/axios.ts";
+import {useRoute} from "vue-router"
+import {ElMessage} from "element-plus";
+
+const token = ref(useRoute().query.token)
+const input = ref(useRoute().query.token)
+const result = ref()
+
+const inputChange = () => {
+  if (input.value != null && input.value != '') {
+    axios.get('/api/token/info', {
+      params: {
+        token: input.value
+      }
+    }).then(res => {
+      result.value = res.data.result
+      token.value = input.value
+      ElMessage({
+        message: '更改成功',
+        type: 'success',
+      })
+    }).catch(error => {
+      ElMessage({
+        message: 'Token输入错误',
+        type: 'error',
+      })
+    })
+  }
+}
+
+const getInfo = () => {
+  if (token.value != null && token.value != '') {
+    axios.get('/api/token/info', {
+      params: {
+        token: token.value
+      }
+    }).then(res => {
+      result.value = res.data.result
+    })
+  }
+}
+
+getInfo()
+setInterval(getInfo, 5000)
+</script>
+
+
+<template>
+  <b>当前Token：</b>
+  <el-input
+      v-model="input"
+      style="width: 240px"
+      placeholder="输入Token"
+      clearable
+      @change="inputChange"
+  />
+
+  <el-divider/>
+  <b>Token信息（每5秒刷新）</b>
+  <el-button type="primary" plain @click="getInfo">手动刷新</el-button>
+  <el-descriptions
+      direction="vertical"
+      :column="4"
+      border
+  >
+    <el-descriptions-item label="去重对象">{{ result?.dedup_object }}</el-descriptions-item>
+    <el-descriptions-item label="上传数据格式">{{ result?.data_format }}</el-descriptions-item>
+    <el-descriptions-item label="去重记录值">{{ result?.dedup_items_number }}</el-descriptions-item>
+    <el-descriptions-item label="Redis中数据条数">{{ result?.cache_list_number }}</el-descriptions-item>
+  </el-descriptions>
+</template>
+
+<style scoped>
+
+</style>

web/src/views/TokenDetailView.vue

@@ -1,14 +1,33 @@
 <script setup lang="ts">
 import {useCounterStore} from "@/stores/counter.ts";
-import {ref} from 'vue'
+import {ref, watch} from 'vue'
 import axios from "@/axios.ts";
 
+
+// 创建响应式引用，用于存储API请求结果
 const result = ref()
 
+// 创建响应式引用，用于存储当前选中的token值
+const value = ref('')
+value.value = useCounterStore().token
+
+// 创建响应式引用，用于存储下拉选项列表
+const options = ref([] as string[])
+
+// 控制删除指定Redis键的确认对话框的显示状态
+const deleteSpecifyDataVisible = ref(false)
+const inputSpecifyData = ref('')
+
+const deleteSpecifyDedupVisible = ref(false)
+const inputSpecifyDedup = ref('')
+const deleteSpecifyRawVisible = ref(false)
+const inputSpecifyRaw = ref('')
+
 const getInfo = () => {
+  if (value.value != '') {
     axios.get('/api/token/info', {
       params: {
-      token: useCounterStore().token
+        token: value.value
       }
     }).then(res => {
       if (res.status == 200) {
@@ -16,47 +35,162 @@ const getInfo = () => {
       }
     })
   }
+}
 
 const deleteDedup = () => {
   axios.delete('/api/token/info', {
     params: {
-      token: useCounterStore().token,
-      dedup_bf: true
+      token: value.value,
+      dedup_bf: "all"
     }
-  })
+  }).then(res => {
     getInfo()
+  })
 }
+
 const deleteRedis = () => {
   axios.delete('/api/token/info', {
     params: {
-      token: useCounterStore().token,
-      cache_list: true
+      token: value.value,
+      cache_list: "all",
     }
-  })
+  }).then(res => {
     getInfo()
+  })
 }
 
 getInfo()
 setInterval(getInfo, 5000)
+
+watch(value, (newValue) => {
+  useCounterStore().token = value.value
+  getInfo()
+})
+
+
+axios.get('/api/token').then(res => {
+  if (res.status == 200) {
+    res.data.result.forEach((item: any) => {
+      options.value.push(item.token)
+    })
+  }
+})
+
+const deleteSpecifyData = () => {
+  axios.delete('/api/token/info', {
+    params: {
+      token: value.value,
+      both_number: inputSpecifyData.value,
+    }
+  }).then(res => {
+    getInfo()
+    deleteSpecifyDataVisible.value = false
+  })
+}
+const deleteSpecifyDedup = () => {
+  axios.delete('/api/token/info', {
+    params: {
+      token: value.value,
+      dedup_bf: inputSpecifyDedup.value,
+    }
+  }).then(res => {
+    getInfo()
+    deleteSpecifyDedupVisible.value = false
+  })
+}
+const deleteSpecifyRaw = () => {
+  axios.delete('/api/token/info', {
+    params: {
+      token: value.value,
+      cache_list: inputSpecifyRaw.value,
+    }
+  }).then(res => {
+    getInfo()
+    deleteSpecifyRawVisible.value = false
+  })
+}
 </script>
 
 
 <template>
-  <p>当前Token：{{ useCounterStore().token }}</p>
-  <el-button type="danger" @click="deleteDedup">删除去重记录值</el-button>
-  <el-button type="danger" @click="deleteRedis">删除Redis数据</el-button>
+  <div v-if="!useCounterStore().isAdmin">
+    <el-alert title="您没有权限访问此页面" type="error" center show-icon/>
+  </div>
+
+
+  <div v-if="useCounterStore().isAdmin">
+    <b>当前Token：</b>
+    <el-select v-model="value" placeholder="选择Token" style="width: 240px">
+      <el-option
+          v-for="item in options"
+          :key="item"
+          :value="item"
+      />
+    </el-select>
+
     <el-divider/>
-  <el-button type="primary" @click="getInfo">手动刷新</el-button>
+
+    <b>Token信息（每5秒刷新）</b>
+    <el-button type="primary" plain @click="getInfo">手动刷新</el-button>
     <el-descriptions
-      :title="'Token信息 - ' + useCounterStore().token+'(每5秒刷新)'"
         direction="vertical"
         :column="4"
         border
     >
-    <el-descriptions-item label="去重对象">{{ result?.dedup_object }}</el-descriptions-item>
-    <el-descriptions-item label="去重记录值">{{ result?.dedup_items_number }}</el-descriptions-item>
-    <el-descriptions-item label="Redis中数据条数">{{ result?.cache_list_number }}</el-descriptions-item>
+      <el-descriptions-item label="去重对象" label-width="150px">
+        <el-tag>{{ result?.dedup_object }}</el-tag>
+      </el-descriptions-item>
+      <el-descriptions-item label="上传数据格式" label-width="150px">
+        {{ result?.data_format }}
+      </el-descriptions-item>
+      <el-descriptions-item label="去重参考值数量" label-width="150px">
+        {{ result?.dedup_items_number }} 条
+      </el-descriptions-item>
+      <el-descriptions-item label="原始数据数量" label-width="150px">
+        {{ result?.cache_list_number }} 条
+      </el-descriptions-item>
     </el-descriptions>
+
+    <p><b>管理</b></p>
+    <el-button type="danger" @click="deleteDedup">删除全部去重参考值</el-button>
+    <el-button type="danger" @click="deleteRedis">删除全部原始数据</el-button>
+    <div style="margin-top: 10px">
+      <el-button type="danger" @click="deleteSpecifyDedupVisible=true">删除指定数量去重参考值</el-button>
+      <el-button type="danger" @click="deleteSpecifyRawVisible=true">删除指定数量原始数据</el-button>
+    </div>
+    <div style="margin-top: 10px">
+      <el-button type="danger" @click="deleteSpecifyDataVisible=true">
+        删除指定数量的数据（去重参考值+原始数据）
+      </el-button>
+    </div>
+
+
+    <!--弹窗输入-->
+    <el-dialog v-model="deleteSpecifyDedupVisible" title="删除指定数量去重参考值" width="400">
+      <el-input v-model="inputSpecifyDedup" style="width: 200px" placeholder="请输入删除数量"/>
+      <template #footer>
+        <el-button type="primary" @click="deleteSpecifyDedup">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
+    <el-dialog v-model="deleteSpecifyRawVisible" title="删除指定数量原始数据" width="400">
+      <el-input v-model="inputSpecifyRaw" style="width: 200px" placeholder="请输入删除数量"/>
+      <template #footer>
+        <el-button type="primary" @click="deleteSpecifyRaw">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
+    <el-dialog v-model="deleteSpecifyDataVisible" title="删除指定数量的数据" width="400">
+      <el-input v-model="inputSpecifyData" style="width: 200px" placeholder="请输入删除数量"/>
+      <template #footer>
+        <el-button type="primary" @click="deleteSpecifyData">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
+  </div>
 </template>
 
 <style scoped>

web/src/views/TokenListView.vue

@@ -1,175 +0,0 @@
-<script setup lang="ts">
-import {ref} from "vue";
-import axios from "@/axios.ts";
-import {ElMessage, ElMessageBox} from 'element-plus'
-
-const tableData = ref([])
-
-axios.get("/api/token").then(res => {
-  tableData.value = res.data.result
-})
-
-const input = ref('')
-const value = ref('')
-const options = [
-  {
-    value: 'uid',
-    label: 'uid',
-  },
-  {
-    value: 'pid',
-    label: 'pid',
-  },
-  {
-    value: 'secid',
-    label: 'secid',
-  },
-  {
-    value: 'dyid',
-    label: 'dyid',
-  },
-  {
-    value: 'comment_id',
-    label: 'comment_id',
-  }
-]
-
-const addToken = () => {
-  axios.post('/api/token', {}, {
-    params: {
-      token: input.value,
-      dedup_object: value.value
-    }
-  }).then(response => {
-    if (response.data.result == "ok") {
-      ElMessage({
-        message: '添加成功',
-        type: 'success',
-      })
-      axios.get('/api/token').then(res => {
-        tableData.value = res.data.result
-      })
-    }
-  }).catch(error => {
-    ElMessage.error(error.response?.data?.error)
-  })
-}
-
-import {useCounterStore} from "@/stores/counter.ts";
-import {useRouter} from "vue-router";
-
-const router = useRouter()
-const viewDetails = (row: any) => {
-  useCounterStore().token = row.token
-  router.push({
-    name: "TokenDetail"
-  })
-}
-
-var rowOut: any
-const dedupObjectVisible = ref(false)
-const dialogDedupObjectVisible = (row: any) => {
-  rowOut = row
-  dedupObjectVisible.value = true
-}
-const updateDedupObject = () => {
-  dedupObjectVisible.value = false
-  axios.put('/api/token', {}, {
-    params: {
-      token: rowOut.token,
-      dedup_object: value.value
-    }
-  }).then(res => {
-    if (res.data.result == "ok") {
-      ElMessage({
-        message: '更改成功',
-        type: 'success',
-      })
-      axios.get('/api/token').then(res => {
-        tableData.value = res.data.result
-      })
-    }
-  }).catch(error => {
-    ElMessage.error(error.response?.data?.error)
-  })
-
-}
-
-const deleteToken = (row: any) => {
-  axios.delete('/api/token', {
-    params: {
-      token: row.token
-    }
-  }).then(res => {
-    if (res.data.result == "ok") {
-      ElMessage({
-        message: '删除成功',
-        type: 'success',
-      })
-      axios.get('/api/token').then(res => {
-        tableData.value = res.data.result
-      })
-    }
-  }).catch(error => {
-    ElMessage.error(error.response?.data?.error)
-  })
-}
-</script>
-
-<template>
-  <!--添加Token-->
-  <el-input v-model="input" style="width: 200px" placeholder="请输入Token名称"/>
-  <el-select v-model="value" placeholder="选择去重对象" style="width: 200px">
-    <el-option
-        v-for="item in options"
-        :key="item.value"
-        :label="item.label"
-        :value="item.value"
-    />
-  </el-select>
-  <el-button type="primary" @click="addToken">添加Token</el-button>
-
-  <!--Token列表-->
-  <el-table :data="tableData" style="width: 100%">
-    <el-table-column prop="token" label="Token" width="180"/>
-    <el-table-column prop="dedup_object" label="去重对象" width="180"/>
-    <el-table-column label="操作">
-      <template #default="scope">
-        <el-button @click="viewDetails(scope.row)">查看详细</el-button>
-        <el-button @click="dialogDedupObjectVisible(scope.row)" type="primary">更改去重对象</el-button>
-        <el-popconfirm
-            width="180"
-            title="确认删除此Token吗"
-            confirm-button-text="确认"
-            cancel-button-text="取消"
-            @confirm="deleteToken(scope.row)"
-        >
-          <template #reference>
-            <el-button type="danger">删除此Token</el-button>
-          </template>
-        </el-popconfirm>
-
-      </template>
-    </el-table-column>
-  </el-table>
-
-  <el-dialog v-model="dedupObjectVisible" title="更改去重对象" width="400">
-    <el-select v-model="value" placeholder="选择去重对象" style="width: 200px">
-      <el-option
-          v-for="item in options"
-          :key="item.value"
-          :label="item.label"
-          :value="item.value"
-      />
-    </el-select>
-    <template #footer>
-      <el-button type="primary" @click="updateDedupObject">
-        确定
-      </el-button>
-    </template>
-  </el-dialog>
-</template>
-
-<style scoped>
-
-</style>

web/src/views/TokenManageView.vue

@@ -0,0 +1,304 @@
+<script setup lang="ts">
+import {ref} from "vue";
+import axios from "@/axios.ts";
+import {ElMessage, ElMessageBox} from 'element-plus'
+import {useCounterStore} from "@/stores/counter.ts";
+import {useRouter} from "vue-router";
+
+const tableData = ref([])
+const input = ref('')
+const value = ref('')
+const dataFormat = ref('')
+const inputPassWord = ref('')
+const router = useRouter()
+var rowOut: any
+const dedupObjectVisible = ref(false)
+const dataFormatVisible = ref(false)
+const inputNotes = ref("")
+const NotesVisible = ref(false)
+const options = [
+  {
+    value: 'uid',
+  },
+  {
+    value: 'secid',
+  },
+  {
+    value: 'pid',
+  },
+  {
+    value: 'comment_id',
+  },
+  {
+    value: 'dyid',
+  }
+]
+
+const dataFormatOptions = [
+  {
+    value: 'uid----secid----pid----comment_id',
+  }, {
+    value: 'uid----secid',
+  }, {
+    value: 'dyid',
+  }
+]
+
+
+axios.get("/api/token").then(res => {
+  tableData.value = res.data.result
+})
+
+const addToken = () => {
+  axios.post('/api/token', {}, {
+    params: {
+      token: input.value,
+      dedup_object: value.value,
+      data_format: dataFormat.value,
+      notes: inputNotes.value,
+    }
+  }).then(response => {
+    if (response.data.result == "ok") {
+      ElMessage({
+        message: '添加成功',
+        type: 'success',
+      })
+      axios.get('/api/token').then(res => {
+        tableData.value = res.data.result
+      })
+    }
+  }).catch(error => {
+    ElMessage.error(error.response?.data?.error)
+  })
+}
+
+const viewDetails = (row: any) => {
+  useCounterStore().token = row.token
+  router.push({
+    name: "TokenDetail",
+  })
+}
+
+const dialogDedupObjectVisible = (row: any) => {
+  rowOut = row
+  dedupObjectVisible.value = true
+}
+
+const updateDedupObject = () => {
+  dedupObjectVisible.value = false
+  axios.put('/api/token', {}, {
+    params: {
+      token: rowOut.token,
+      dedup_object: value.value,
+      data_format: rowOut.data_format,
+      notes: rowOut.notes,
+    }
+  }).then(res => {
+    if (res.data.result == "ok") {
+      ElMessage({
+        message: '更改成功',
+        type: 'success',
+      })
+      axios.get('/api/token').then(res => {
+        tableData.value = res.data.result
+      })
+    }
+  }).catch(error => {
+    ElMessage.error(error.response?.data?.error)
+  })
+}
+
+const dialogDataFormatVisible = (row: any) => {
+  rowOut = row
+  dataFormatVisible.value = true
+}
+
+const updateDataFormat = () => {
+  dataFormatVisible.value = false
+  axios.put('/api/token', {}, {
+    params: {
+      token: rowOut.token,
+      dedup_object: rowOut.dedup_object,
+      data_format: dataFormat.value,
+      notes: rowOut.notes,
+    }
+  }).then(res => {
+    if (res.data.result == "ok") {
+      ElMessage({
+        message: '更改成功',
+        type: 'success',
+      })
+      axios.get('/api/token').then(res => {
+        tableData.value = res.data.result
+      })
+    }
+  }).catch(error => {
+    ElMessage.error(error.response?.data?.error)
+  })
+}
+
+const dialogNotesVisible = (row: any) => {
+  rowOut = row
+  NotesVisible.value = true
+}
+
+const updateNotes = () => {
+  NotesVisible.value = false
+  axios.put('/api/token', {}, {
+    params: {
+      token: rowOut.token,
+      dedup_object: rowOut.dedup_object,
+      data_format: rowOut.data_format,
+      notes: inputNotes.value
+    }
+  }).then(res => {
+    if (res.data.result == "ok") {
+      ElMessage({
+        message: '更改成功',
+        type: 'success',
+      })
+      axios.get('/api/token').then(res => {
+        tableData.value = res.data.result
+      })
+    }
+  }).catch(error => {
+    ElMessage.error(error.response?.data?.error)
+  })
+}
+
+const deleteToken = (row: any) => {
+  axios.delete('/api/token', {
+    params: {
+      token: row.token
+    }
+  }).then(res => {
+    if (res.data.result == "ok") {
+      ElMessage({
+        message: '删除成功',
+        type: 'success',
+      })
+      axios.get('/api/token').then(res => {
+        tableData.value = res.data.result
+      })
+    }
+  }).catch(error => {
+    ElMessage.error(error.response?.data?.error)
+  })
+}
+
+const checkPassword = () => {
+  if (inputPassWord.value == "haha") {
+    ElMessage({
+      message: '密码正确',
+      type: 'success',
+    })
+    useCounterStore().isAdmin = true
+  } else {
+    ElMessage.error('密码错误')
+  }
+}
+</script>
+
+<template>
+  <!--非管理员-->
+  <div v-if="!useCounterStore().isAdmin" style="margin: auto auto; max-width: 400px">
+    <el-alert title="您没有权限访问此页面，输入管理员密码" type="error" :closable="false" show-icon/>
+    <p></p>
+    <el-input v-model="inputPassWord" style="width: 400px" placeholder="请输入管理员密码" @change="checkPassword"/>
+    <p></p>
+    <el-button type="primary" @click="checkPassword">确认</el-button>
+  </div>
+
+  <!--管理员-->
+  <div v-if="useCounterStore().isAdmin">
+    <!--添加Token-->
+    <el-input v-model="input" style="width: 150px" placeholder="请输入Token名称"/>
+    <el-select v-model="value" placeholder="选择去重对象" style="width: 150px">
+      <el-option
+          v-for="item in options"
+          :key="item.value"
+          :value="item.value"
+      />
+    </el-select>
+    <el-select v-model="dataFormat" placeholder="选择数据格式" style="width: 280px">
+      <el-option
+          v-for="item in dataFormatOptions"
+          :key="item.value"
+          :value="item.value"
+      />
+    </el-select>
+    <el-input v-model="inputNotes" style="width: 200px" placeholder="请输入备注"/>
+    <el-button type="primary" @click="addToken">添加Token</el-button>
+
+    <!--Token列表-->
+    <el-table :data="tableData" stripe style="width: 100%">
+      <el-table-column prop="token" label="Token" width="150"/>
+      <el-table-column prop="dedup_object" label="去重对象" width="150"/>
+      <el-table-column prop="data_format" label="上传数据格式" width="280"/>
+      <el-table-column prop="notes" label="备注" width="200"/>
+      <el-table-column label="操作">
+        <template #default="scope">
+          <el-button @click="viewDetails(scope.row)">查看详细</el-button>
+          <el-button @click="dialogDedupObjectVisible(scope.row)" type="primary">更改去重对象</el-button>
+          <el-button @click="dialogDataFormatVisible(scope.row)" type="primary">更改数据格式</el-button>
+          <el-button @click="dialogNotesVisible(scope.row)" type="primary">更改备注</el-button>
+          <el-popconfirm
+              width="180"
+              title="确认删除此Token吗"
+              confirm-button-text="确认"
+              cancel-button-text="取消"
+              @confirm="deleteToken(scope.row)"
+          >
+            <template #reference>
+              <el-button type="danger">删除此Token</el-button>
+            </template>
+          </el-popconfirm>
+
+        </template>
+      </el-table-column>
+    </el-table>
+
+    <el-dialog v-model="dedupObjectVisible" title="更改去重对象" width="400">
+      <el-select v-model="value" placeholder="选择去重对象" style="width: 200px">
+        <el-option
+            v-for="item in options"
+            :key="item.value"
+            :value="item.value"
+        />
+      </el-select>
+      <template #footer>
+        <el-button type="primary" @click="updateDedupObject">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
+
+    <el-dialog v-model="dataFormatVisible" title="更改数据格式" width="400">
+      <el-select v-model="dataFormat" placeholder="选择数据格式" style="width: 280px">
+        <el-option
+            v-for="item in dataFormatOptions"
+            :key="item.value"
+            :value="item.value"
+        />
+      </el-select>
+      <template #footer>
+        <el-button type="primary" @click="updateDataFormat">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
+
+    <el-dialog v-model="NotesVisible" title="更改备注" width="400">
+      <el-input v-model="inputNotes" style="width: 200px" placeholder="请输入备注"/>
+      <template #footer>
+        <el-button type="primary" @click="updateNotes">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
+  </div>
+</template>
+
+<style scoped>
+
+</style>