refactor(tool): 优化文件处理逻辑和性能

.gitea/workflows/build_tool.yaml

@@ -13,9 +13,7 @@ jobs:
       - name: 安装Go（镜像）
         run: |
           # 使用国内镜像下载 Go
-          export GO_MIRROR_URL="https://mirrors.aliyun.com/golang"
+          wget https://mirrors.aliyun.com/golang/go1.25.1.linux-amd64.tar.gz -O go.tar.gz
-          wget $GO_MIRROR_URL/go1.25.1.linux-amd64.tar.gz -O go.tar.gz
-          
           # 解压并设置环境变量
           sudo rm -rf /usr/local/go
           sudo tar -C /usr/local -xzf go.tar.gz
@@ -23,11 +21,6 @@ jobs:
         env:
           GOROOT: /usr/local/go
 
-#      - name: 安装Go
-#        uses: actions/setup-go@v5
-#        with:
-#          go-version-file: "go.mod"
-
       - name: 构建上传工具
         run: |
           go env -w CGO_ENABLED=0 \

controller/tokenController.go

@@ -3,7 +3,9 @@ package controller
 import (
 	"dypid/db"
 	"dypid/global"
+	"fmt"
 	"net/http"
+	"strconv"
 
 	"github.com/gin-gonic/gin"
 )
@@ -18,7 +20,7 @@ func CreateTokenHandler(c *gin.Context) {
 		Token       string `form:"token" binding:"required"`
 		DedupObject string `form:"dedup_object" binding:"required"`
 		DataFormat  string `form:"data_format" binding:"required"`
-		Notes       string `form:"notes" binding:"required"`
+		Notes       string `form:"notes"`
 	}{}
 	if err := c.ShouldBindQuery(&input); err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
@@ -41,7 +43,7 @@ func UpdateTokenHandler(c *gin.Context) {
 		Token       string `form:"token" binding:"required"`
 		DedupObject string `form:"dedup_object" binding:"required"`
 		DataFormat  string `form:"data_format" binding:"required"`
-		Notes       string `form:"notes" binding:"required"`
+		Notes       string `form:"notes"`
 	}{}
 	if err := c.ShouldBindQuery(&input); err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "参数不能为空 " + err.Error()})
@@ -115,8 +117,8 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 	//解析输入数据
 	input := struct {
 		Token     string `form:"token" binding:"required"`
-		DedupBF   bool   `form:"dedup_bf"`
+		DedupBF   string `form:"dedup_bf"`
-		CacheList bool   `form:"cache_list"`
+		CacheList string `form:"cache_list"`
 	}{}
 	if err := c.ShouldBindQuery(&input); err != nil {
 		c.JSON(http.StatusBadRequest, gin.H{"error": "Token不能为空"})
@@ -131,12 +133,30 @@ func DeleteTokenInfoHandler(c *gin.Context) {
 	}
 
 	//删除去重对象
-	if input.DedupBF {
+	switch input.DedupBF {
+	case "":
+	case "all":
 		keys := global.RDB.Keys(global.RCtx, "dedup:"+input.Token+":*").Val()
 		global.RDB.Del(global.RCtx, keys...)
+	default:
+		_, err := strconv.Atoi(input.DedupBF)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "dedup_bf设置错误 " + err.Error()})
+			return
 		}
-	if input.CacheList {
+	}
+	switch input.CacheList {
+	case "":
+	case "all":
 		global.RDB.Del(global.RCtx, "list:"+input.Token)
+	default:
+		i, err := strconv.Atoi(input.CacheList)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "cache_list设置错误 " + err.Error()})
+			return
+		}
+		fmt.Println(-i)
+		global.RDB.LTrim(global.RCtx, "list:"+input.Token, 1, int64(-i))
 	}
 
 	//输出信息

tool/update-dypid.go

@@ -48,7 +48,6 @@ func main() {
 
 	//检测./upload
 	fmt.Println("程序启动成功，正在检测txt文件")
-	//os.Mkdir("./upload", os.ModePerm)
 	for {
 		files, err := getTxtFiles("./")
 		if err != nil {
@@ -62,16 +61,19 @@ func main() {
 				wg.Add(1)
 				go func() {
 					processFile(filePath)
-					os.Remove(filePath)
+					err := os.Truncate(filePath, 0)
+					if err != nil {
+						fmt.Println("清空文件失败：", err)
+					}
 					wg.Done()
 				}()
 			}
 			wg.Wait()
+		}
+		time.Sleep(time.Minute)
+	}
+}
 
-		}
-		time.Sleep(2 * time.Second)
-	}
-}
 func uploadDataToServer(data string) error {
 	params := url.Values{}
 	params.Set("token", viper.GetString("token"))
@@ -88,10 +90,8 @@ func uploadDataToServer(data string) error {
 }
 
 // 获取目录中的所有txt文件
-func getTxtFiles(dir string) ([]string, error) {
+func getTxtFiles(dir string) (txtFiles []string, err error) {
-	var txtFiles []string
+	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
-
-	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -151,6 +151,7 @@ func processFile(filePath string) {
 
 	if err := scanner.Err(); err != nil {
 		fmt.Printf("读取文件 %s 错误: %v\n", filePath, err)
+		return
 	}
 
 	fmt.Printf("文件【%s】处理完成，共处理 %d 行数据\n", filePath, lineCount)

web/src/views/TokenDetailView.vue

@@ -34,11 +34,12 @@ const deleteDedup = () => {
     getInfo()
   })
 }
+
 const deleteRedis = () => {
   axios.delete('/api/token/info', {
     params: {
       token: value.value,
-      cache_list: true
+      cache_list: "all",
     }
   }).then(res => {
     getInfo()
@@ -66,6 +67,21 @@ axios.get('/api/token').then(res => {
     })
   }
 })
+
+const deleteSpecifyRedisVisible = ref(false)
+const inputSpecifyRedis = ref('')
+
+const deleteSpecifyRedis = () => {
+  axios.delete('/api/token/info', {
+    params: {
+      token: value.value,
+      cache_list: inputSpecifyRedis.value,
+    }
+  }).then(res => {
+    getInfo()
+    deleteSpecifyRedisVisible.value = false
+  })
+}
 </script>
 
 
@@ -100,7 +116,17 @@ axios.get('/api/token').then(res => {
 
     <p><b>管理</b></p>
     <el-button type="danger" @click="deleteDedup">删除去重记录值</el-button>
-    <el-button type="danger" @click="deleteRedis">删除Redis数据</el-button>
+    <el-button type="danger" @click="deleteRedis">删除全部Redis数据</el-button>
+    <el-button type="danger" @click="deleteSpecifyRedisVisible=true">删除指定数量Redis数据</el-button>
+
+    <el-dialog v-model="deleteSpecifyRedisVisible" title="删除指定数量Redis数据" width="400">
+      <el-input v-model="inputSpecifyRedis" style="width: 200px" placeholder="请输入删除数量"/>
+      <template #footer>
+        <el-button type="primary" @click="deleteSpecifyRedis">
+          确定
+        </el-button>
+      </template>
+    </el-dialog>
   </div>
 </template>