feat(uploader): 添加文件上传配置和临时文件处理功能

- 新增 ClearFilesNoPrompt 配置项用于控制是否提示清空文件
- 实现临时文件目录(tmp)管理，优先处理未上传完的文件
- 添加文件复制功能，支持快速零拷贝技术
- 实现文件清空提示机制，支持用户确认操作
- 优化文件上传流程，添加进度跟踪和状态更新
- 过滤掉大小为0的文件，避免无效上传
- 修改数据结构名称提升代码可读性

frontend/package.json

@@ -11,6 +11,7 @@
     "type-check": "vue-tsc --build"
   },
   "dependencies": {
+    "@element-plus/icons-vue": "^2.3.2",
     "element-plus": "^2.13.7",
     "vue": "^3.5.32"
   },

frontend/package.json.md5

@@ -1 +1 @@
-aa914e6b4676ee4621ced7ad6d81c58c
+05225657934ff66d822c925754c951bf

frontend/pnpm-lock.yaml

@@ -8,6 +8,9 @@ importers:
 
   .:
     dependencies:
+      '@element-plus/icons-vue':
+        specifier: ^2.3.2
+        version: 2.3.2(vue@3.5.33(typescript@6.0.3))
       element-plus:
         specifier: ^2.13.7
         version: 2.13.7(typescript@6.0.3)(vue@3.5.33(typescript@6.0.3))

frontend/src/App.vue

@@ -1,11 +1,12 @@
 <script lang="ts" setup>
-import {ref, nextTick, computed} from 'vue'
-import {ElMessage} from 'element-plus'
-import {ElMessageBox} from 'element-plus'
-import {SelectPath, GetConfig, WriteConfig, StartUpload, StopUpload} from '../wailsjs/go/main/App';
+import {computed, nextTick, ref} from 'vue'
+import {ElMessage, ElMessageBox} from 'element-plus'
+import ConfirmClearDialog from './components/ConfirmClearDialog.vue'
+import {GetConfig, SelectPath, StartUpload, StopUpload, WriteConfig} from '../wailsjs/go/main/App';
 import {config} from "../wailsjs/go/models.ts";
-import Config = config.Config;
 import {EventsOn, LogPrint} from "../wailsjs/runtime";
+import {configModel} from "@/model.ts";
+import Config = config.Config;
 
 // const serverUrl = ref('')
 const token = ref('')
@@ -19,6 +20,10 @@ const logOutput = ref<string[]>([])
 const logContentRef = ref<HTMLElement>()
 const logRoll = ref(true)
 
+const clearDialogVisible = ref(false)
+const filesToClear = ref<string[]>([])
+const noPromptClear = ref(false)
+
 interface FileProgress {
   name: string
   uploaded: number
@@ -41,7 +46,7 @@ const sortedProgressList = computed(() => {
 })
 
 const addLog = (msg: string) => {
-  logOutput.value.push(`[${new Date().toLocaleString()}]` + msg)
+  logOutput.value.push(`[${new Date().toLocaleString()}] ` + msg)
   nextTick(() => {
     if (logContentRef.value && logRoll.value) {
       logContentRef.value.scrollTop = logContentRef.value.scrollHeight
@@ -52,9 +57,9 @@ const addLog = (msg: string) => {
 const selectDirectory = () => {
   // ElMessage.info('请手动输入检测目录路径')
   SelectPath().then((path) => {
-    if (path){
+    if (path) {
       checkDir.value = path
-    }else {
+    } else {
       ElMessage.warning('未选择目录，不更改配置')
     }
   })
@@ -99,17 +104,17 @@ const clearLog = () => {
 // const writeServerUrl =() => {
 //   WriteConfig("url", serverUrl.value)
 // }
-const writeToken =() => {
-  WriteConfig("token", token.value)
+const writeToken = () => {
+  WriteConfig(configModel.Token, token.value)
 }
 const writeCheckDir = () => {
-  WriteConfig("check-dir", checkDir.value)
+  WriteConfig(configModel.CheckDir, checkDir.value)
 }
 const writeConcurrentFiles = () => {
-  WriteConfig("handle-file-count", concurrentFiles.value)
+  WriteConfig(configModel.HandleFileCount, concurrentFiles.value)
 }
-const writeUploadThreads =() => {
-  WriteConfig("thread-count", uploadThreads.value)
+const writeUploadThreads = () => {
+  WriteConfig(configModel.ThreadCount, uploadThreads.value)
 }
 // const writeAutoStart = () => {
 //   WriteConfig("is-run-on-start", autoStart.value)
@@ -124,6 +129,7 @@ try {
     concurrentFiles.value = config.handle_file_count
     uploadThreads.value = config.thread_count
     // autoStart.value = config.is_run_on_start
+    noPromptClear.value = config.clear_files_no_prompt
 
     LogPrint(`[${new Date().toLocaleString()}] 配置已加载`)
   })
@@ -131,24 +137,34 @@ try {
   console.log(e)
 }
 
-EventsOn("is-run", (run) => {
-  isRunning.value = run
-})
-EventsOn("progress", (progress) => {
-  progressList.value = progress
-})
-EventsOn("log", (msg) => {
-  addLog(msg)
-})
+try {
+  EventsOn("is-run", (run) => {
+    isRunning.value = run
+  })
+  EventsOn("progress", (progress) => {
+    progressList.value = progress
+  })
+  EventsOn("log", (msg) => {
+    addLog(msg)
+  })
+  EventsOn("clear-files", (files) => {
+    filesToClear.value = files
+    if (!noPromptClear.value) {
+      clearDialogVisible.value = true
+    }
+  })
+} catch (e) {
+  console.log(e)
+}
 </script>
 
 <template>
   <div class="container">
     <div class="left-panel">
-<!--      <div class="form-item">-->
-<!--        <label>服务器地址</label>-->
-<!--        <el-input v-model="serverUrl" placeholder="请输入服务器地址" :disabled="isRunning" @change="writeServerUrl()"/>-->
-<!--      </div>-->
+      <!--      <div class="form-item">-->
+      <!--        <label>服务器地址</label>-->
+      <!--        <el-input v-model="serverUrl" placeholder="请输入服务器地址" :disabled="isRunning" @change="writeServerUrl()"/>-->
+      <!--      </div>-->
 
       <div class="form-item">
         <label>Token</label>
@@ -165,17 +181,19 @@ EventsOn("log", (msg) => {
 
       <div class="form-item">
         <label>同时处理文件数</label>
-        <el-input-number v-model="concurrentFiles" :min="1" :max="100" :disabled="isRunning" @change="writeConcurrentFiles()"/>
+        <el-input-number v-model="concurrentFiles" :min="1" :max="100" :disabled="isRunning"
+                         @change="writeConcurrentFiles()"/>
       </div>
 
       <div class="form-item">
         <label>单文件上传线程</label>
-        <el-input-number v-model="uploadThreads" :min="1" :max="100" :disabled="isRunning" @change="writeUploadThreads()"/>
+        <el-input-number v-model="uploadThreads" :min="1" :max="100" :disabled="isRunning"
+                         @change="writeUploadThreads()"/>
       </div>
 
-<!--      <div class="form-item">-->
-<!--        <el-checkbox v-model="autoStart" label="运行时自动启动上传" size="large" :disabled="isRunning" @change="writeAutoStart()"/>-->
-<!--      </div>-->
+      <!--      <div class="form-item">-->
+      <!--        <el-checkbox v-model="autoStart" label="运行时自动启动上传" size="large" :disabled="isRunning" @change="writeAutoStart()"/>-->
+      <!--      </div>-->
 
       <div class="form-item">
         <label>上传进度</label>
@@ -205,6 +223,11 @@ EventsOn("log", (msg) => {
       </div>
     </div>
   </div>
+
+  <ConfirmClearDialog
+      v-model:visible="clearDialogVisible"
+      :file-list="filesToClear"
+  />
 </template>
 
 <style scoped>

frontend/src/components/ConfirmClearDialog.vue

@@ -0,0 +1,124 @@
+<script lang="ts" setup>
+import {ref} from 'vue'
+import {WriteConfig} from "../../wailsjs/go/main/App";
+import {configModel} from "@/model.ts";
+import {EventsEmit} from "../../wailsjs/runtime";
+import {InfoFilled} from '@element-plus/icons-vue'
+
+const props = defineProps<{
+  visible: boolean
+  fileList: string[]
+}>()
+
+const emit = defineEmits<{
+  (e: 'update:visible', value: boolean): void
+  // (e: 'confirm', dontShowAgain: boolean): void
+}>()
+
+const dontShowAgain = ref(false)
+
+const handleClose = () => {
+  emit('update:visible', false)
+  EventsEmit('confirm-clear-files', false)
+}
+
+const handleConfirm = () => {
+  // emit('confirm', dontShowAgain.value)
+  emit('update:visible', false)
+  WriteConfig(configModel.ClearFilesNoPrompt, dontShowAgain.value)
+  EventsEmit('confirm-clear-files', true)
+}
+</script>
+
+<template>
+  <el-dialog
+      :model-value="visible"
+      width="600px"
+      :close-on-click-modal="false"
+      @update:model-value="(val: boolean) => emit('update:visible', val)"
+      @close="handleClose"
+      align-center
+  >
+    <template #header>
+      是否确认清空上传文件
+    </template>
+    <div class="hint-text">以下文件将会被清空并移动到tmp文件夹进行上传，您是否确认</div>
+    <div class="dialog-content">
+      <div class="file-list">
+        <div v-for="(file, index) in fileList" :key="index" class="file-item">
+          {{ file }}
+        </div>
+        <div v-if="fileList.length === 0" class="empty-text">暂无文件</div>
+      </div>
+    </div>
+    <template #footer>
+      <div class="dialog-footer">
+        <div class="dialog-footer-checkbox-container">
+          <el-checkbox v-model="dontShowAgain" label="下次清空文件不再弹出此弹窗确认"/>
+          <el-tooltip content="此弹窗会在首次运行时弹出，您可以选择下次不再弹出此弹窗">
+            <el-icon>
+              <InfoFilled/>
+            </el-icon>
+          </el-tooltip>
+        </div>
+
+        <div>
+          <el-button @click="handleClose">取消</el-button>
+          <el-button type="primary" @click="handleConfirm">确认</el-button>
+        </div>
+      </div>
+    </template>
+  </el-dialog>
+</template>
+
+<style scoped>
+.dialog-footer-checkbox-container {
+  display: flex;
+  align-items: center;
+  gap: 5px;
+}
+
+.dialog-footer-checkbox-container .el-icon {
+  color: #909399;
+}
+
+.dialog-content {
+  padding: 10px 0;
+}
+
+.hint-text {
+  font-size: 12px;
+  color: #909399;
+}
+
+.file-list {
+  max-height: 350px;
+  overflow-y: auto;
+  border: 1px solid #e4e7ed;
+  border-radius: 4px;
+  padding: 8px;
+}
+
+.file-item {
+  padding: 6px 8px;
+  font-size: 14px;
+  color: #606266;
+  border-bottom: 1px solid #f0f0f0;
+}
+
+.file-item:last-child {
+  border-bottom: none;
+}
+
+.empty-text {
+  text-align: center;
+  color: #909399;
+  padding: 20px 0;
+}
+
+.dialog-footer {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+}
+</style>

frontend/src/model.ts

@@ -0,0 +1,9 @@
+export enum configModel {
+    Url = "url",
+    Token = "token",
+    ThreadCount = "thread-count",
+    HandleFileCount = "handle-file-count",
+    IsRunOnStart = "is-run-on-start",
+    CheckDir = "check-dir",
+    ClearFilesNoPrompt = "clear-files-no-prompt",
+}

frontend/wailsjs/go/models.ts

@@ -7,6 +7,7 @@ export namespace config {
 	    handle_file_count: number;
 	    is_run_on_start: boolean;
 	    check_dir: string;
+	    clear_files_no_prompt: boolean;
 	
 	    static createFrom(source: any = {}) {
 	        return new Config(source);
@@ -20,6 +21,7 @@ export namespace config {
 	        this.handle_file_count = source["handle_file_count"];
 	        this.is_run_on_start = source["is_run_on_start"];
 	        this.check_dir = source["check_dir"];
+	        this.clear_files_no_prompt = source["clear_files_no_prompt"];
 	    }
 	}
 

internal/config/config.go

@@ -9,35 +9,38 @@ import (
 )
 
 type Config struct {
-	Url             string `json:"url" mapstructure:"url"`
-	Token           string `json:"token" mapstructure:"token"`
-	ThreadCount     int    `json:"thread_count" mapstructure:"thread-count"`
-	HandleFileCount int    `json:"handle_file_count" mapstructure:"handle-file-count"`
-	IsRunOnStart    bool   `json:"is_run_on_start" mapstructure:"is-run-on-start"`
-	CheckDir        string `json:"check_dir" mapstructure:"check-dir"`
+	Url                string `json:"url" mapstructure:"url"`
+	Token              string `json:"token" mapstructure:"token"`
+	ThreadCount        int    `json:"thread_count" mapstructure:"thread-count"`
+	HandleFileCount    int    `json:"handle_file_count" mapstructure:"handle-file-count"`
+	IsRunOnStart       bool   `json:"is_run_on_start" mapstructure:"is-run-on-start"`
+	CheckDir           string `json:"check_dir" mapstructure:"check-dir"`
+	ClearFilesNoPrompt bool   `json:"clear_files_no_prompt" mapstructure:"clear-files-no-prompt"`
 }
 
 var APPConfig Config
 var configMu sync.Mutex
 
 const (
-	Url             = "url"
-	Token           = "token"
-	ThreadCount     = "thread-count"
-	HandleFileCount = "handle-file-count"
-	IsRunOnStart    = "is-run-on-start"
-	CheckDir        = "check-dir"
+	Url                = "url"
+	Token              = "token"
+	ThreadCount        = "thread-count"
+	HandleFileCount    = "handle-file-count"
+	IsRunOnStart       = "is-run-on-start"
+	CheckDir           = "check-dir"
+	ClearFilesNoPrompt = "clear-files-no-prompt"
 )
 
 func InitConfig() {
 	// 设置默认配置
 	defaultConfig := Config{
-		Url:             "http://127.0.0.1:8080",
-		Token:           "",
-		ThreadCount:     10,
-		HandleFileCount: 25,
-		IsRunOnStart:    false,
-		CheckDir:        "",
+		Url:                "http://127.0.0.1:8080",
+		Token:              "",
+		ThreadCount:        10,
+		HandleFileCount:    25,
+		IsRunOnStart:       false,
+		CheckDir:           "",
+		ClearFilesNoPrompt: false,
 	}
 	viper.SetDefault(Url, defaultConfig.Url)
 	viper.SetDefault(Token, defaultConfig.Token)
@@ -45,6 +48,7 @@ func InitConfig() {
 	viper.SetDefault(HandleFileCount, defaultConfig.HandleFileCount)
 	viper.SetDefault(IsRunOnStart, defaultConfig.IsRunOnStart)
 	viper.SetDefault(CheckDir, defaultConfig.CheckDir)
+	viper.SetDefault(ClearFilesNoPrompt, defaultConfig.ClearFilesNoPrompt)
 
 	//设置配置文件名和路径 ./config.toml
 	viper.AddConfigPath(".")

internal/uploader/uploader.go

@@ -7,6 +7,7 @@ import (
 	"dypid-client/internal/api"
 	"dypid-client/internal/config"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -20,7 +21,7 @@ import (
 	"golang.org/x/sync/errgroup"
 )
 
-type Task struct {
+type fileInfo struct {
 	FilePath  string
 	FileLines int
 }
@@ -80,28 +81,92 @@ func StartUpload(ctx context.Context, logChan *chan string) {
 }
 
 func uploadData(ctx context.Context, logChan *chan string) {
-	// 获取检测目录
-	var path = "./"
-	if config.APPConfig.CheckDir != "" {
-		path = config.APPConfig.CheckDir
-	}
-
-	//获取文件列表
-	files, err := getTxtFiles(path)
-	if err != nil {
-		AddLog(logChan, "获取文件列表失败："+err.Error())
-		return
-	}
-	if files == nil {
-		return
-	}
-
 	start := time.Now()
 
+	// 获取检测目录
+	var checkPath = "./"
+	if config.APPConfig.CheckDir != "" {
+		checkPath = config.APPConfig.CheckDir
+	}
+
+	//要上传的文件路径字符串数组
+	var files []string
+
+	//先检测tmp目录有没有残余文件
+	os.Mkdir("./tmp", os.ModePerm)
+	tmpFiles, err := getTxtFiles("./tmp")
+	if err != nil {
+		AddLog(logChan, "获取 tmp 文件列表失败："+err.Error())
+	}
+
+	//tmp有文件，优先上传（else：tmp没文件扫描指定文件夹，并复制文件到tmp）
+	if tmpFiles != nil {
+		AddLog(logChan, "当前 tmp 目录下还有未上传完成文件，将优先上传 tmp 目录文件")
+		files = tmpFiles
+	} else {
+		//tmp没文件，扫描指定文件夹
+		f, err := getTxtFiles(checkPath)
+		if err != nil {
+			AddLog(logChan, "获取文件列表失败："+err.Error())
+			return
+		}
+
+		//指定文件夹没文件，退出函数
+		if f == nil {
+			return
+		}
+
+		//是否向用户提示清空文件，并复制文件到tmp
+		if config.APPConfig.ClearFilesNoPrompt {
+			//不用提示直接复制文件到tmp
+			for _, p := range f {
+				err := copyFile(p, "./tmp/"+filepath.Base(p))
+				if err != nil {
+					AddLog(logChan, "复制文件失败："+err.Error())
+				} else {
+					files = append(files, "./tmp/"+filepath.Base(p))
+					err := os.Truncate(p, 0)
+					if err != nil {
+						AddLog(logChan, "清空文件失败："+err.Error())
+					}
+				}
+			}
+		} else {
+			//提示用户
+			wailsruntime.EventsEmit(ctx, "clear-files", f)
+
+			confirm := make(chan bool)
+
+			wailsruntime.EventsOn(ctx, "confirm-clear-files", func(optionalData ...any) {
+				confirm <- optionalData[0].(bool)
+			})
+
+			if <-confirm {
+				for _, p := range f {
+					err := copyFile(p, "./tmp/"+filepath.Base(p))
+					if err != nil {
+						AddLog(logChan, "复制文件失败："+err.Error())
+					} else {
+						files = append(files, "./tmp/"+filepath.Base(p))
+						err := os.Truncate(p, 0)
+						if err != nil {
+							AddLog(logChan, "清空文件失败："+err.Error())
+						}
+					}
+				}
+			} else {
+				AddLog(logChan, "已取消上传，1分钟后再运行")
+				return
+			}
+		}
+	}
+
 	//检测到文件
 	//统计文件行数
-	fileLines := make(map[string]int)
+	var fInfo = make(map[string]fileInfo)
+
 	AddLog(logChan, fmt.Sprintf("正在统计 %v 个文件行数", len(files)))
+
 	isAllEmpty := true
 	for _, filePath := range files {
 		select {
@@ -124,8 +189,13 @@ func uploadData(ctx context.Context, logChan *chan string) {
 			if lineCount == 0 {
 				continue
 			}
-			fileLines[filepath.Base(filePath)] = lineCount
+			fInfo[filepath.Base(filePath)] = fileInfo{
+				FilePath:  filePath,
+				FileLines: lineCount,
+			}
+
 			isAllEmpty = false
+
 			AddLog(logChan, fmt.Sprintf("%s 文件行数：%v", filepath.Base(filePath), lineCount))
 		}
 	}
@@ -135,20 +205,24 @@ func uploadData(ctx context.Context, logChan *chan string) {
 		return
 	}
 
+	//刷新文件上传进度
 	progress.Clear()
-
-	//添加文件上传任务参数（文件路径，文件行数）
-	var tasks []Task
-	for fileName, lines := range fileLines {
-		tasks = append(tasks, Task{FilePath: path + "/" + fileName, FileLines: lines})
-		progress.Store(fileName, Progress{FileName: fileName, Total: lines, Uploaded: 0, Percentage: 0})
+	for fileName, info := range fInfo {
+		progress.Store(fileName,
+			Progress{
+				FileName:   fileName,
+				Total:      info.FileLines,
+				Uploaded:   0,
+				Percentage: 0,
+			},
+		)
 	}
 
 	// 使用 errgroup 控制同时处理的文件数，并开始上传文件任务
 	g, egctx := errgroup.WithContext(ctx)
 	g.SetLimit(config.APPConfig.HandleFileCount) // 设置同时处理文件数
-	// 执行所有任务
-	for _, task := range tasks {
+	// 执行文件上传任务参数（文件路径，文件行数）
+	for fileName, info := range fInfo {
 		select {
 		case <-egctx.Done():
 			return
@@ -158,18 +232,18 @@ func uploadData(ctx context.Context, logChan *chan string) {
 				case <-egctx.Done():
 					return egctx.Err()
 				default:
-					AddLog(logChan, "正在上传文件："+filepath.Base(task.FilePath))
+					AddLog(logChan, "正在上传文件："+fileName)
 
-					processFile(egctx, logChan, task.FilePath, task.FileLines)
+					processFile(egctx, logChan, info.FilePath, info.FileLines)
 
 					select {
 					case <-egctx.Done():
 						return egctx.Err()
 					default:
-						//上传完成，清空文件
-						err := os.Truncate(task.FilePath, 0)
+						//上传完成，删除缓存文件
+						err := os.Remove(info.FilePath)
 						if err != nil {
-							AddLog(logChan, "清空文件失败："+err.Error())
+							AddLog(logChan, "删除缓存文件失败："+err.Error())
 						}
 						return nil
 					}
@@ -190,7 +264,33 @@ func uploadData(ctx context.Context, logChan *chan string) {
 	}
 }
 
-// 获取目录中的所有txt文件
+// copyFile 快速拷贝文件 src -> dst
+func copyFile(src, dst string) error {
+	// 打开源文件
+	sourceFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sourceFile.Close()
+
+	// 创建目标文件
+	destFile, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer destFile.Close()
+
+	// 核心：最快拷贝，底层使用操作系统零拷贝技术
+	_, err = io.Copy(destFile, sourceFile)
+	if err != nil {
+		return err
+	}
+
+	// 强制刷入磁盘，保证数据完整
+	return destFile.Sync()
+}
+
+// 获取目录中的所有txt文件（文件大小为0的不返回）
 func getTxtFiles(dir string) (txtFiles []string, err error) {
 	err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -204,7 +304,9 @@ func getTxtFiles(dir string) (txtFiles []string, err error) {
 
 		// 检查文件扩展名是否为.txt
 		if strings.ToLower(filepath.Ext(path)) == ".txt" {
-			txtFiles = append(txtFiles, path)
+			if info.Size() != 0 {
+				txtFiles = append(txtFiles, path)
+			}
 		}
 
 		return nil